Commit aaeefb7f authored by Tomas Farago's avatar Tomas Farago
Browse files

Merge branch '0.4' of https://gitlab.esrf.fr/tomotools/nxtomomill into 0.4

parents 7f432fba 6ec9a5b8
......@@ -70,7 +70,7 @@ doc:
script:
- python -m unittest nxtomomill.test.suite -v
test:python3.5-stretch-pyqt5:
test:python3.7-stretch-pyqt5:
image: python:3.7-buster
<<: *test_linux_template
......
......@@ -5,14 +5,19 @@ Change Log
0.4.0: 2020/11/09
-----------------
* requires h5py >= 3
* utils:
* add `change_image_key_control` function to modify frame type inplace
* add `add_dark_flat_nx_file` function to add dark or flat in an existing `NXTomo` entry
* h5_to_nx:
* add management of 'proposal file': handle External / SoftLink
* insure relative path is given when converting the file
* magnified_pixel_size will not be write anymore. If a magnified / sample pixel size is discover then this will be saved as the 'pixel_size'.
* add an option to display_advancement or not.
* converter
* h5_to_nx:
* add management of 'proposal file': handle External / SoftLink
* insure relative path is given when converting the file
* magnified_pixel_size will not be write anymore. If a magnified / sample pixel size is discover then this will be saved as the 'pixel_size'.
* add an option to display_advancement or not.
* split "zseries" according to z value
* add NXdata information to display detector/data from root as image
* move format version to 1.0
* app:
* add patch-nx application to modify an existing `NXTomo`
......@@ -22,6 +27,7 @@ Change Log
* warning if we try to convert a file containing some `NXTomo` entry
* create directories for output file if necessary
* check write rights on output file
* split "zseries" according to z value
* miscellaneous::
* adopt 'black' coding style
......
......@@ -83,3 +83,49 @@ Those are some examples of usage:
* modify frame to `dark` type if any
* modify frame to `invalid` if any
Concrete example of modifying flat field on an .nx file
-------------------------------------------------------
In this example flat field frames at from acquisition A has been mess up.
NXTomo file is name original_acquiA.nx. Entry name is entry000A
And we want to replace them by flat field from acquisition B (master file is acquiB.nx). Entry name is entry000B
This is one way to proceed to replace flat field frames:
1. go to acquisition A folder containing the acquiA.nx file
.. code-block:: bash
cd [path_to_acquisitionA_folder]/acquisitionA
2. copy the original .nx file. Modification are in place. This is safer to copy the file.
.. code-block:: bash
cp original_acquiA.nx acquiA.nx
3. invalidate the flat field frames made at start. In the case let say that we want to invalid frames from 20 to 40 included:
.. code-block:: bash
nxtomomill patch-nx set83_tomo_black_drum_LPJ01_6p5p_20N_0001_0000_patch.nx entry0000 --invalid-frames 20:41
4. check that the invalidation of frames worked properly using silx view for example
5. get the silx url you want to link as the new flat field. Syntax is `silx://[file_path]?path=[data_path]&slice=[slices]`
for example here we want to link frames 2000 to 2021 from acquiB.nx
So the url looks like: silx://[folder_to_acquiB]/acquiB.nx?path=/entry000B/instrument/detector/data&slices=2000:2021
6. then patch flat from patch-nx command and the 'flats-at-start' option:
.. code-block:: bash
nxtomomill patch-nx acquiA.nx entry000A --flats-at-start "silx://[folder_to_acquiB]/acquiB.nx?path=/entry000B/instrument/detector/data&slices=2000:2021"
.. warning::
when you provide the url make sure you use `"` or `'` characters. Otherwise in this case the command will be executed as a background task and slices will be ignored.
It will also try to link it with the full dataset at `/entry000B/instrument/detector/data`
7. check that your dataset is complete (using silx view for example)
......@@ -51,6 +51,15 @@ An acquisition file can contain several sequence (so several acquisition).
You can ask the converter to keep all the acquisition into a single file using the '--single-file' option.
Input type
----------
z-series
""""""""
z-series are handled by nxtomomill since 0.4. It will create one entry per 'z' found.
Settings
--------
......@@ -110,5 +119,7 @@ nxtomomill tomoh52nx --help
--set-params [SET_PARAMS [SET_PARAMS ...]]
Allow manual definition of some parameters. Valid
parameters (and expected input unit) are: energy
(kev).
(kev). Should be added at the end of the command line
because will try to cover all text set after this
option.
```
\ No newline at end of file
......@@ -191,6 +191,12 @@ def main(argv):
help="Define the set of frames to be mark as alignment. "
"" + _INFO_FRAME_INPUT,
)
parser.add_argument(
"--embed-data",
default=False,
action="store_true",
help="Embed data from url in the file if not already inside",
)
options = parser.parse_args(argv[1:])
......@@ -271,6 +277,7 @@ def main(argv):
flats_start=flat_start_url,
darks_end=darks_end_url,
flats_end=flat_end_url,
embed_data=options.embed_data,
logger=_logger,
)
......
......@@ -108,20 +108,19 @@ def main(argv):
"--z_trans_key", default=EDF_Z_TRANS, help="z translation key in EDF HEADER"
)
print("******set up***********")
options = parser.parse_args(argv[1:])
conv = utils.get_tuple_of_keys_from_cmd
file_keys = converter.EDFFileKeys(
motor_mne_key=conv(options.motor_mne_key),
motor_pos_key=conv(options.motor_pos_key),
motor_mne_key=options.motor_mne_key,
motor_pos_key=options.motor_pos_key,
ref_names=conv(options.refs_name_keys),
to_ignore=conv(options.ignore_file_containing),
rot_angle_key=conv(options.rot_angle_key),
rot_angle_key=options.rot_angle_key,
dark_names=conv(options.dark_names),
x_trans_key=conv(options.x_trans_key),
y_trans_key=conv(options.y_trans_key),
z_trans_key=conv(options.z_trans_key),
x_trans_key=options.x_trans_key,
y_trans_key=options.y_trans_key,
z_trans_key=options.z_trans_key,
)
input_dir = options.scan_path
......
......@@ -49,6 +49,7 @@ from nxtomomill.settings import (
H5_REF_TITLES,
H5_DARK_TITLES,
H5_INIT_TITLES,
H5_ZSERIE_INIT_TITLES,
)
from nxtomomill.converter import h5_to_nx, H5FileKeys, H5ScanTitles
from tomoscan.esrf.hdf5scan import HDF5TomoScan
......@@ -127,7 +128,7 @@ def main(argv):
parser = argparse.ArgumentParser(
description="convert data acquired as "
"hdf5 from bliss to nexus "
"`NXtomo` classes"
"`NXtomo` classes. For `zseries` it will create one entry per `z`"
)
parser.add_argument("input_file_path", help="master file of the " "acquisition")
parser.add_argument("output_file", help="output .nx or .h5 file")
......@@ -153,6 +154,12 @@ def main(argv):
action="store_true",
default=False,
)
parser.add_argument(
"--entries",
help="Specify (root) entries to be converted. By default it will try "
"to convert all existing entries.",
default=None,
)
parser.add_argument(
"--raises-error",
help="Raise errors if some data are not met instead of providing some"
......@@ -171,21 +178,25 @@ def main(argv):
)
parser.add_argument(
"--x_trans_keys",
"--x-trans-keys",
default=",".join(H5_X_TRANS_KEYS),
help="x translation key in bliss HDF5 file",
)
parser.add_argument(
"--y_trans_keys",
"--y-trans-keys",
default=",".join(H5_Y_TRANS_KEYS),
help="y translation key in bliss HDF5 file",
)
parser.add_argument(
"--z_trans_keys",
"--z-trans-keys",
default=",".join(H5_Z_TRANS_KEYS),
help="z translation key in bliss HDF5 file",
)
parser.add_argument(
"--valid_camera_names",
"--valid-camera-names",
default=None,
help="Valid NXDetector dataset name to be considered. Otherwise will"
"try to deduce them from NX_class attibute (value should be"
......@@ -193,44 +204,63 @@ def main(argv):
)
parser.add_argument(
"--rot_angle_keys",
"--rot-angle-keys",
default=",".join(H5_ROT_ANGLE_KEYS),
help="Valid dataset name for rotation angle",
)
parser.add_argument(
"--acq_expo_time_keys",
"--acq-expo-time-keys",
default=",".join(H5_ACQ_EXPO_TIME_KEYS),
help="Valid dataset name for acquisition exposure time",
)
parser.add_argument(
"--x_pixel_size_key", default=H5_X_PIXEL_SIZE, help="X pixel size key to read"
"--x_pixel_size_key",
"--x-pixel-size-key",
default=H5_X_PIXEL_SIZE,
help="X pixel size key to read",
)
parser.add_argument(
"--y_pixel_size_key", default=H5_Y_PIXEL_SIZE, help="Y pixel size key to read"
"--y_pixel_size_key",
"--y-pixel-size-key",
default=H5_Y_PIXEL_SIZE,
help="Y pixel size key to read",
)
# scan titles
parser.add_argument(
"--init_titles",
"--init-titles",
default=",".join(H5_INIT_TITLES),
help="Titles corresponding to init scans",
)
parser.add_argument(
"--init_zserie_titles",
"--init-zserie-titles",
default=",".join(H5_ZSERIE_INIT_TITLES),
help="Titles corresponding to zserie init scans",
)
parser.add_argument(
"--dark_titles",
"--dark-titles",
default=",".join(H5_DARK_TITLES),
help="Titles corresponding to dark scans",
)
parser.add_argument(
"--ref_titles",
"--ref-titles",
default=",".join(H5_REF_TITLES),
help="Titles corresponding to ref scans",
)
parser.add_argument(
"--proj_titles",
"--proj-titles",
default=",".join(H5_PROJ_TITLES),
help="Titles corresponding to projection scans",
)
parser.add_argument(
"--align_titles",
"--align-titles",
default=",".join(H5_ALIGNMENT_TITLES),
help="Titles corresponding to alignment scans",
)
......@@ -240,7 +270,9 @@ def main(argv):
nargs="*",
help="Allow manual definition of some parameters. "
"Valid parameters (and expected input unit) "
"are: {}.".format(_getPossibleInputParams()),
"are: {}. Should be added at the end of the command line because "
"will try to cover all text set after this "
"option.".format(_getPossibleInputParams()),
)
options = parser.parse_args(argv[1:])
......@@ -273,6 +305,7 @@ def main(argv):
)
scan_titles = H5ScanTitles(
init_titles=conv(options.init_titles),
init_zserie_titles=conv(options.init_zserie_titles),
dark_titles=conv(options.dark_titles),
ref_titles=conv(options.ref_titles),
proj_titles=conv(options.proj_titles),
......@@ -285,6 +318,11 @@ def main(argv):
else:
callback_det_sel = None
if options.entries is not None:
entries = conv(options.entries)
else:
entries = None
h5_to_nx(
input_file_path=options.input_file_path,
output_file=options.output_file,
......@@ -298,6 +336,7 @@ def main(argv):
raise_error_if_issue=options.raises_error,
detector_sel_callback=callback_det_sel,
ask_before_overwrite=not options.overwrite,
entries=entries,
)
exit(0)
......
This diff is collapsed.
......@@ -66,13 +66,16 @@ H5_ACQ_EXPO_TIME_KEYS = ("acq_expo_time",)
H5_INIT_TITLES = (
"pcotomo:basic",
"tomo:basic",
"tomo:zseries",
"tomo:fullturn",
"sequence_of_scans",
"tomo:halfturn",
)
"""if a scan starts by one of those string then is considered as
initialization scan"""
H5_ZSERIE_INIT_TITLES = ("tomo:zseries",)
"""specific titles for zserie. Will extend H5_INIT_TITLES"""
H5_DARK_TITLES = ("dark images", "dark")
"""if a scan starts by one of those string then is considered as
dark scan"""
......
......@@ -36,6 +36,8 @@ import numpy
import os
from nxtomomill import converter
from nxtomomill.test.utils.bliss import MockBlissAcquisition
from tomoscan.esrf.hdf5scan import HDF5TomoScan
from glob import glob
class TestH5ToNxConverter(unittest.TestCase):
......@@ -217,6 +219,44 @@ class TestH5ToNxConverter(unittest.TestCase):
raise_error_if_issue=True,
)
def test_z_series_conversion(self):
"""Test conversion of a zseries bliss (mock) acquisition"""
bliss_mock = MockBlissAcquisition(
n_sample=1,
n_sequence=1,
n_scan_per_sequence=10,
n_darks=5,
n_flats=5,
with_nx_detector_attr=True,
output_dir=self.folder,
detector_name="frelon1",
acqui_type="zseries",
z_values=(1, 2, 3),
)
self.assertTrue(len(bliss_mock.samples), 1)
sample = bliss_mock.samples[0]
self.assertTrue(os.path.exists(sample.sample_file))
output_file = sample.sample_file.replace(".h5", ".nx")
res = converter.h5_to_nx(
input_file_path=sample.sample_file,
output_file=output_file,
single_file=False,
request_input=False,
entries=None,
file_extension=None,
raise_error_if_issue=False,
)
# insure the 4 files are generated: master file and one per z
files = glob(os.path.dirname(sample.sample_file) + "/*.nx")
self.assertEqual(len(files), 4)
# try to create HDF5TomoScan from those to insure this is valid
# and check z values for example
for res_tuple in res:
scan = HDF5TomoScan(scan=res_tuple[0], entry=res_tuple[1])
if hasattr(scan, "z_translation"):
self.assertTrue(scan.z_translation is not None)
class TestDetectorDetection(unittest.TestCase):
"""
......
......@@ -40,6 +40,7 @@ from nxtomomill.utils import add_dark_flat_nx_file
from nxtomomill.utils import change_image_key_control
from tomoscan.esrf.hdf5scan import HDF5TomoScan
from nxtomomill.utils import ImageKey
from silx.io.utils import h5py_read_dataset
class BaseTestAddDarkAndFlats(unittest.TestCase):
......@@ -65,7 +66,7 @@ class BaseTestAddDarkAndFlats(unittest.TestCase):
data_path = "/".join(
(self._simple_nx.entry, "instrument", "detector", "data")
)
self._raw_data = h5s[data_path][()]
self._raw_data = h5py_read_dataset(h5s[data_path])
nx_with_vds_path = os.path.join(self.tmpdir, "case_with_vds")
self._nx_with_virtual_dataset = MockHDF5(
scan_path=nx_with_vds_path,
......@@ -216,7 +217,7 @@ class BaseTestAddDarkAndFlats(unittest.TestCase):
with h5py.File(source_file, mode="r") as o_h5s:
if new_path in h5s:
del h5s[new_path]
h5s[new_path] = o_h5s[old_path][()]
h5s[new_path] = h5py_read_dataset(o_h5s[old_path])
elif source_file == target_file:
h5s[new_path] = h5py.SoftLink(old_path)
else:
......@@ -321,7 +322,9 @@ class TestAddDarkAtStart(BaseTestAddDarkAndFlats):
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.start_dark.shape[0]
)
......@@ -418,7 +421,9 @@ class TestAddFlatAtStart(BaseTestAddDarkAndFlats):
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.start_flat.shape[0]
)
......@@ -474,17 +479,21 @@ class TestAddFlatAtEnd(BaseTestAddDarkAndFlats):
scan.entry, "instrument", "detector", "image_key_control"
)
numpy.testing.assert_array_equal(
h5s[img_key_control_path][-2:][()], [0, 1]
h5py_read_dataset(h5s[img_key_control_path][-2:]), [0, 1]
)
img_key_path = os.path.join(
scan.entry, "instrument", "detector", "image_key"
)
numpy.testing.assert_array_equal(h5s[img_key_path][-2:][()], [0, 1])
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[img_key_path][-2:]), [0, 1]
)
# test rotation angle and count_time
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.end_flat.shape[0]
)
......@@ -544,17 +553,21 @@ class TestAddFlatAtEnd(BaseTestAddDarkAndFlats):
scan.entry, "instrument", "detector", "image_key_control"
)
numpy.testing.assert_array_equal(
h5s[img_key_control_path][-2:][()], [0, 1]
h5py_read_dataset(h5s[img_key_control_path][-2:]), [0, 1]
)
img_key_path = os.path.join(
scan.entry, "instrument", "detector", "image_key"
)
numpy.testing.assert_array_equal(h5s[img_key_path][-2:][()], [0, 1])
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[img_key_path][-2:]), [0, 1]
)
# test rotation angle and count_time
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.end_flat.shape[0]
)
......@@ -717,7 +730,8 @@ class TestCompleteAddFlatAndDark(BaseTestAddDarkAndFlats):
)
numpy.testing.assert_array_almost_equal(h5s[rotation_angle_dataset][-3], 89)
numpy.testing.assert_array_almost_equal(
h5s[rotation_angle_dataset][1:4][()], numpy.array([10, 11, 12])
h5py_read_dataset(h5s[rotation_angle_dataset][1:4]),
numpy.array([10, 11, 12]),
)
numpy.testing.assert_array_almost_equal(
h5s[rotation_angle_dataset][-2], h5s[rotation_angle_dataset][-1]
......
......@@ -45,6 +45,9 @@ class MockBlissAcquisition:
create one serie of n flats after dark if any
:param str output_dir: will contain the proposal file and one folder per
sequence.
:param str acqui_type: acquisition type. Can be "basic" or "zseries"
:param Iterable z_values: if acqui_type is zseries then users should
provide the serie of values for z (one per stage)
"""
def __init__(
......@@ -57,6 +60,8 @@ class MockBlissAcquisition:
output_dir,
with_nx_detector_attr=True,
detector_name="pcolinux",
acqui_type="basic",
z_values=None,
):
self.__folder = output_dir
if not os.path.exists(output_dir):
......@@ -70,8 +75,8 @@ class MockBlissAcquisition:
sample_dir = os.path.join(self.path, dir_name)
os.mkdir(sample_dir)
sample_file = os.path.join(sample_dir, dir_name + ".h5")
self.__samples.append(
_BlissSample(
if acqui_type == "basic":
acqui_tomo = _BlissBasicTomo(
sample_dir=sample_dir,
sample_file=sample_file,
n_sequence=n_sequence,
......@@ -81,7 +86,23 @@ class MockBlissAcquisition:
with_nx_detector_attr=with_nx_detector_attr,
detector_name=detector_name,
)
)
elif acqui_type == "zseries":
if z_values is None:
raise ValueError("for zseries z_values should be provided")
acqui_tomo = _BlissZseriesTomo(
sample_dir=sample_dir,
sample_file=sample_file,
n_sequence=n_sequence,
n_scan_per_sequence=n_scan_per_sequence,
n_darks=n_darks,
n_flats=n_flats,
with_nx_detector_attr=with_nx_detector_attr,
detector_name=detector_name,
z_values=z_values,
)
else:
raise NotImplementedError("")
self.__samples.append(acqui_tomo)
@property
def samples(self):
......@@ -114,20 +135,20 @@ class _BlissSample:
detector_name,
with_nx_detector_attr=True,
):
self.__with_nx_detector_attr = with_nx_detector_attr
self.__sample_dir = sample_dir
self.__sample_file = sample_file
self.__n_sequence = n_sequence
self.__n_scan_per_seq = n_scan_per_sequence
self.__n_darks = n_darks
self.__n_flats = n_flats
self.__scan_folders = []
self._with_nx_detector_attr = with_nx_detector_attr
self._sample_dir = sample_dir
self._sample_file = sample_file
self._n_sequence = n_sequence
self._n_scan_per_seq = n_scan_per_sequence
self._n_darks = n_darks
self._n_flats = n_flats
self._scan_folders = []
self._index = 1
self.__detector_name = detector_name
self.__det_width = 64
self.__det_height = 64
self.__n_frame_per_scan = 10
self.__energy = 19.0
self._detector_name = detector_name
self._det_width = 64
self._det_height = 64
self._n_frame_per_scan = 10
self._energy = 19.0
for i_sequence in range(n_sequence):
self.add_sequence()
......@@ -136,6 +157,9 @@ class _BlissSample:
self._index += 1
return idx
def get_main_entry_title(self):
raise NotImplementedError("Base class")
@staticmethod
def get_title(scan_type):
if scan_type == "dark":
......@@ -147,108 +171,97 @@ class _BlissSample:
else:
raise ValueError("Not implemented")
def add_sequence(self):
# reserve the index for the 'initialization' sequence. No scan folder
# will be created for this one.
seq_ini_index = self.get_next_free_index()
def create_entry_and_energy(self, seq_ini_index):
# add sequence init information
with h5py.File(self.sample_file, mode="a") as h5f:
seq_node = h5f.require_group(str(seq_ini_index) + ".1")
seq_node.attrs["NX_class"] = u"NXentry"
seq_node["title"] = "tomo:fullturn"
seq_node["title"] = self.get_main_entry_title()