From feb0279ec201fd1c25ada87dbbfeb64cd9a693e0 Mon Sep 17 00:00:00 2001 From: Henri Payno Date: Wed, 13 Apr 2022 15:35:14 +0200 Subject: [PATCH] h5py - FrameAppender: improve handling of the data type. Could happen that the dtype was set to None (float64 by default) which was incoherent with the type. But due to the structure of it this look slike this was not bringing any major issue. --- .../hdf5/acquisition/standardacquisition.py | 5 +- .../converter/hdf5/test/test_hdf5converter.py | 64 +++++++++++++++++++ nxtomomill/test/utils/bliss.py | 19 +++++- nxtomomill/utils/frameappender.py | 2 + nxtomomill/utils/h5pyutils.py | 4 +- 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/nxtomomill/converter/hdf5/acquisition/standardacquisition.py b/nxtomomill/converter/hdf5/acquisition/standardacquisition.py index 313c23d..67646d8 100644 --- a/nxtomomill/converter/hdf5/acquisition/standardacquisition.py +++ b/nxtomomill/converter/hdf5/acquisition/standardacquisition.py @@ -289,7 +289,10 @@ class StandardAcquisition(BaseAcquisition): # "treated file". Because .name if the name on the 'target' # file of the virtual dataset v_source = h5py.VirtualSource( - camera_dataset_url.file_path(), data_dataset_path, data_dataset.shape + camera_dataset_url.file_path(), + data_dataset_path, + data_dataset.shape, + dtype=self._data_type, ) self._virtual_sources.append(v_source) self._virtual_sources_len.append(n_frame) diff --git a/nxtomomill/converter/hdf5/test/test_hdf5converter.py b/nxtomomill/converter/hdf5/test/test_hdf5converter.py index ccc34be..18afe23 100644 --- a/nxtomomill/converter/hdf5/test/test_hdf5converter.py +++ b/nxtomomill/converter/hdf5/test/test_hdf5converter.py @@ -54,6 +54,7 @@ from nxtomomill.io.framegroup import FrameGroup from nxtomomill.utils import Format import subprocess from nxtomomill.nexus.nxtomo import NXtomo +import pytest def url_has_been_copied(file_path: str, url: DataUrl): @@ -403,6 +404,7 @@ class TestXRDCTConversion(unittest.TestCase): self.folder = tempfile.mkdtemp() self.config = TomoHDF5Config() self.config.format = Format.XRD_CT + self.frame_data_type = numpy.float32 def tearDown(self) -> None: shutil.rmtree(self.folder) @@ -424,6 +426,7 @@ class TestXRDCTConversion(unittest.TestCase): output_dir=self.folder, detector_name="pilatus", acqui_type="xrd-ct", + frame_data_type=self.frame_data_type, ) for sample in bliss_mock.samples: self.assertTrue(os.path.exists(sample.sample_file)) @@ -441,6 +444,7 @@ class TestXRDCTConversion(unittest.TestCase): for _, entry_node in h5s.items(): self.assertTrue("instrument/detector/data" in entry_node) dataset = entry_node["instrument/detector/data"] + self.assertEqual(dataset.dtype, self.frame_data_type) # check virtual dataset are relative and valid self.assertTrue(dataset.is_virtual) for vs in dataset.virtual_sources(): @@ -466,6 +470,7 @@ class TestStandardAcqConversionWithExternalUrls(unittest.TestCase): :param int n_flats: number of frame per flats :param int n_darks: number of frame per dark """ + self.frame_data_type = numpy.uint16 bliss_mock = MockBlissAcquisition( n_sample=1, n_sequence=1, @@ -476,6 +481,7 @@ class TestStandardAcqConversionWithExternalUrls(unittest.TestCase): output_dir=output_dir, detector_name="pcolinux", y_rot=True, + frame_data_type=self.frame_data_type, ) return bliss_mock.samples[0].sample_file @@ -533,6 +539,7 @@ class TestStandardAcqConversionWithExternalUrls(unittest.TestCase): with h5py.File(self.config.output_file, mode="r") as h5f: dataset = h5f["entry0000/instrument/detector/data"] self.assertTrue(dataset.is_virtual) + self.assertEqual(dataset.dtype, self.frame_data_type) # check the `data` virtual dataset is valid # if the link fail then all values are zeros url = tuple(scan.projections.values())[0] @@ -739,6 +746,7 @@ class TestZSeriesConversionWithExternalUrls(unittest.TestCase): def setUp(self) -> None: super().setUp() self.folder = tempfile.mkdtemp() + self.frame_data_type = numpy.uint64 self.config = TomoHDF5Config() self.config.output_file = os.path.join(self.folder, "output.nx") @@ -755,6 +763,7 @@ class TestZSeriesConversionWithExternalUrls(unittest.TestCase): detector_name=camera_name, acqui_type="zseries", z_values=(1, 2, 3), + frame_data_type=self.frame_data_type, ) self._zseries_1_file = bliss_mock.samples[0].sample_file @@ -769,6 +778,7 @@ class TestZSeriesConversionWithExternalUrls(unittest.TestCase): detector_name=camera_name, acqui_type="zseries", z_values=(4, 5, 6), + frame_data_type=self.frame_data_type, ) self._zseries_2_file = bliss_mock.samples[0].sample_file @@ -869,6 +879,7 @@ class TestZSeriesConversionWithExternalUrls(unittest.TestCase): numpy.testing.assert_array_equal( proj_entry_2["instrument/frelon/data"], dataset[20:30] ) + self.assertEqual(dataset.dtype, self.frame_data_type) class TestH5ToNxFrmCommandLine(unittest.TestCase): @@ -887,6 +898,7 @@ class TestH5ToNxFrmCommandLine(unittest.TestCase): with_nx_detector_attr=True, output_dir=self.folder, detector_name="pcolinux", + frame_data_type=numpy.uint32, ) sample = bliss_mock.samples[0] self.input_file = sample.sample_file @@ -934,6 +946,7 @@ class TestH5ToNxFrmCommandLine(unittest.TestCase): for vs_info in dataset.virtual_sources(): self.assertTrue(dataset.is_virtual) self.assertEqual(os.path.realpath(vs_info.file_name), self.output_file) + assert dataset.dtype == numpy.uint32 with h5py.File(output_file, "r") as h5f: assert "/entry0000/instrument/diode" not in h5f @@ -941,3 +954,54 @@ class TestH5ToNxFrmCommandLine(unittest.TestCase): # insure an nxtomo can be created from it nx_tomo = NXtomo("").load(output_file, "entry0000") assert nx_tomo.energy is not None + + +input_types = ( + numpy.uint8, + numpy.uint16, + numpy.uint32, + numpy.uint64, + numpy.float16, + numpy.float32, + numpy.int16, + numpy.int32, + numpy.int64, +) + + +@pytest.mark.parametrize("input_type", input_types) +def test_simple_conversion(input_type): + """test simple conversion from different frame data type""" + with tempfile.TemporaryDirectory() as root_dir: + bliss_mock = MockBlissAcquisition( + n_sample=1, + n_sequence=1, + n_scan_per_sequence=10, + n_darks=5, + n_flats=5, + with_nx_detector_attr=True, + output_dir=root_dir, + detector_name="my_detector", + frame_data_type=input_type, + ) + sample = bliss_mock.samples[0] + assert os.path.exists(sample.sample_file) + + config = TomoHDF5Config() + config.output_file = sample.sample_file.replace(".h5", ".nx") + config.valid_camera_names = ("my_detec*",) + config.input_file = sample.sample_file + config.single_file = True + config.request_input = False + config.raises_error = True + config.rotation_angle_keys = ("hrsrot",) + + converter.from_h5_to_nx(configuration=config) + # insure only one file is generated + assert os.path.exists(config.output_file) + # insure data is here + with h5py.File(config.output_file, mode="r") as h5s: + for _, entry_node in h5s.items(): + assert "instrument/detector/data" in entry_node + dataset = entry_node["instrument/detector/data"] + assert dataset.dtype == input_type diff --git a/nxtomomill/test/utils/bliss.py b/nxtomomill/test/utils/bliss.py index 2f1800c..ba7c9bb 100644 --- a/nxtomomill/test/utils/bliss.py +++ b/nxtomomill/test/utils/bliss.py @@ -68,6 +68,7 @@ class MockBlissAcquisition: nb_loop=None, nb_tomo=None, with_rotation_motor_info=True, + frame_data_type=numpy.uint16, ): self.__folder = output_dir if not os.path.exists(output_dir): @@ -101,6 +102,7 @@ class MockBlissAcquisition: detector_name=detector_name, y_rot=y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) elif acqui_type == "pcotomo": acqui_tomo = _BlissPCOTomo( @@ -116,6 +118,7 @@ class MockBlissAcquisition: with_rotation_motor_info=with_rotation_motor_info, nb_loop=nb_loop, nb_tomo=nb_tomo, + frame_data_type=frame_data_type, ) elif acqui_type == "zseries": if z_values is None: @@ -132,6 +135,7 @@ class MockBlissAcquisition: z_values=z_values, y_rot=y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) elif acqui_type == "xrd-ct": acqui_tomo = _BlissXRD_CT( @@ -145,6 +149,7 @@ class MockBlissAcquisition: detector_name=detector_name, y_rot=y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) else: raise NotImplementedError("") @@ -182,6 +187,7 @@ class _BlissSample: with_nx_detector_attr=True, y_rot=False, with_rotation_motor_info=True, + frame_data_type=numpy.uint16, ): self._with_nx_detector_attr = with_nx_detector_attr self._sample_dir = sample_dir @@ -201,9 +207,14 @@ class _BlissSample: self._pixel_size = (0.0065, 0.0066) self._y_rot = y_rot self._with_rotation_motor_info = with_rotation_motor_info + self._frame_data_type = frame_data_type for i_sequence in range(n_sequence): self.add_sequence() + @property + def frame_data_type(self): + return self._frame_data_type + def get_next_free_index(self): idx = self._index self._index += 1 @@ -296,7 +307,7 @@ class _BlissSample: data = data.reshape( (self._tomo_n * nb_tomo * nb_loop), self._det_height, self._det_width ) - data = data.astype(numpy.uint16) + data = data.astype(self.frame_data_type) det_path_1 = "/".join(("instrument", self._detector_name)) det_grp = seq_node.require_group(det_path_1) det_grp["data"] = data @@ -399,6 +410,7 @@ class _BlissPCOTomo(_BlissSample): with_rotation_motor_info=True, nb_loop=1, nb_tomo=1, + frame_data_type=numpy.uint16, ): self.nb_loop = nb_loop self.nb_tomo = nb_tomo @@ -413,6 +425,7 @@ class _BlissPCOTomo(_BlissSample): with_nx_detector_attr, y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) def get_main_entry_title(self): @@ -479,6 +492,7 @@ class _BlissZseriesTomo(_BlissSample): with_nx_detector_attr=True, y_rot=False, with_rotation_motor_info=True, + frame_data_type=numpy.uint16, ): self._z_values = z_values super().__init__( @@ -492,6 +506,7 @@ class _BlissZseriesTomo(_BlissSample): with_nx_detector_attr=with_nx_detector_attr, y_rot=y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) def get_main_entry_title(self): @@ -534,6 +549,7 @@ class _BlissXRD_CT(_BlissSample): with_nx_detector_attr=True, y_rot=False, with_rotation_motor_info=True, + frame_data_type=numpy.uint16, ): """XRD-CT are scan with only projection. Projections are store in the init group. Data can be store under 1.1 but also 1.2...""" @@ -548,6 +564,7 @@ class _BlissXRD_CT(_BlissSample): with_nx_detector_attr=with_nx_detector_attr, y_rot=y_rot, with_rotation_motor_info=with_rotation_motor_info, + frame_data_type=frame_data_type, ) def get_main_entry_title(self): diff --git a/nxtomomill/utils/frameappender.py b/nxtomomill/utils/frameappender.py index eedaef1..44cd9e7 100644 --- a/nxtomomill/utils/frameappender.py +++ b/nxtomomill/utils/frameappender.py @@ -207,6 +207,7 @@ class FrameAppender: path_or_dataset=vds_file_path, name=new_virtual_source.name, shape=new_virtual_source.shape, + dtype=new_virtual_source.dtype, ) new_virtual_source.sel = new_virtual_source_sel @@ -346,6 +347,7 @@ class FrameAppender: path_or_dataset=vds_file_path, name=virtual_source.name, shape=virtual_source.shape, + dtype=virtual_source.dtype, ) virtual_source_rel_path.sel = virtual_source.sel layout[:] = virtual_source_rel_path diff --git a/nxtomomill/utils/h5pyutils.py b/nxtomomill/utils/h5pyutils.py index 6914c83..de00109 100644 --- a/nxtomomill/utils/h5pyutils.py +++ b/nxtomomill/utils/h5pyutils.py @@ -70,7 +70,9 @@ def from_data_url_to_virtual_source(url: DataUrl) -> tuple: original_data_shape[-1], ) - vs = h5py.VirtualSource(url.file_path(), url.data_path(), shape=vs_shape) + vs = h5py.VirtualSource( + url.file_path(), url.data_path(), shape=vs_shape, dtype=data_type + ) if url.data_slice() is not None: vs.sel = selection.select(original_data_shape, url.data_slice()) -- GitLab