standardacquisition.py 39.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2015-2020 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/

"""
module to define a standard tomography acquisition (made by bliss)
"""

__authors__ = [
    "H. Payno",
]
__license__ = "MIT"
34
__date__ = "14/02/2022"
35
36


37
from datetime import datetime
38
39

from nxtomomill.utils.utils import str_datetime_to_numpy_datetime64
40
from .baseacquisition import BaseAcquisition
41
from nxtomomill.nexus.nxsource import SourceType
payno's avatar
payno committed
42
from nxtomomill.io.acquisitionstep import AcquisitionStep
43
from .baseacquisition import EntryReader
44
from .utils import deduce_machine_electric_current, get_entry_type
45
46
47
48
49
50
from .utils import guess_nx_detector
from .utils import get_nx_detectors
from nxtomomill.utils import ImageKey
from tomoscan.unitsystem import metricsystem
from silx.io.utils import h5py_read_dataset
import h5py
51
from silx.io.url import DataUrl
52
from typing import Optional, Union
53
from tomoscan.unitsystem import ElectricCurrentSystem
Tomas Farago's avatar
Tomas Farago committed
54
55

try:
Henri Payno's avatar
PEP8    
Henri Payno committed
56
    import hdf5plugin  # noqa F401
Tomas Farago's avatar
Tomas Farago committed
57
58
except ImportError:
    pass
59
60
61
62
import logging
import fnmatch
import numpy
import os
63
from nxtomomill.io.config import TomoHDF5Config
64
from nxtomomill.nexus.nxtomo import NXtomo
65

66
67
68
69

_logger = logging.getLogger(__name__)


payno's avatar
payno committed
70
class StandardAcquisition(BaseAcquisition):
71
    """
72
73
74
75
    Class to collect information from a bliss - hdf scan (see https://bliss.gitlab-pages.esrf.fr/fscan).
    Once all data is collected a set of NXtomo will be created.
    Those NXtomo instances will then be pass to the plugins to allow users to modify.
    Then NXtomo instances will be saved to disk.
76
77
78

    :param DataUrl root_url: url of the acquisition. Can be None if
                             this is the initialization entry
79
80
    :param TomoHDF5Config configuration: configuration to use to collect raw data and generate outputs
    :param Optional[Function] detector_sel_callback: possible callback to retrieve missing information
81
82
83
84
    """

    def __init__(
        self,
85
        root_url: Union[DataUrl, None],
86
        configuration: TomoHDF5Config,
87
        detector_sel_callback,
Henri Payno's avatar
Henri Payno committed
88
        start_index,
89
90
    ):
        super().__init__(
91
            root_url=root_url,
92
            configuration=configuration,
93
            detector_sel_callback=detector_sel_callback,
Henri Payno's avatar
Henri Payno committed
94
            start_index=start_index,
95
        )
96
        self._nx_tomos = [NXtomo("/")]
Henri Payno's avatar
Henri Payno committed
97
98
99
100
101
102
103
104
        self._image_key_control = None
        self._rotation_angle = None
        """list of rotation angles"""
        self._x_translation = None
        """x_translation"""
        self._y_translation = None
        """y_translation"""
        self._z_translation = None
105

Henri Payno's avatar
Henri Payno committed
106
107
        self._virtual_sources = None
        self._acq_expo_time = None
108
109
        self._copied_dataset = {}
        "register dataset copied. Key if the original location as" "DataUrl.path. Value is the DataUrl it has been moved to"
Henri Payno's avatar
Henri Payno committed
110
        self._known_machine_electric_current = None
111
112
113
        # store all registred amchine electric current
        self._frames_timestamp = None
        # try to deduce time stamp of each frame
114

Henri Payno's avatar
Henri Payno committed
115
116
117
    def get_expected_nx_tomo(self):
        return 1

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    @property
    def image_key_control(self):
        return self._image_key_control

    @property
    def rotation_angle(self):
        return self._rotation_angle

    @property
    def x_translation(self):
        return self._x_translation

    @property
    def y_translation(self):
        return self._y_translation

    @property
    def z_translation(self):
        return self._z_translation

    @property
    def n_frames(self):
        return self._n_frames

142
    @property
143
144
    def n_frames_actual_bliss_scan(self):
        return self._n_frames_actual_bliss_scan
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    @property
    def dim_1(self):
        return self._dim_1

    @property
    def dim_2(self):
        return self._dim_2

    @property
    def data_type(self):
        return self._data_type

    @property
    def expo_time(self):
        return self._acq_expo_time
161

162
    @property
Henri Payno's avatar
Henri Payno committed
163
    def known_machine_electric_current(self) -> Optional[dict]:
164
165
166
        """
        Return the dict of all know machine electric current. Key is the time stamp, value is the electric current
        """
Henri Payno's avatar
Henri Payno committed
167
        return self._known_machine_electric_current
168

169
170
171
172
173
174
175
176
    @property
    def is_xrd_ct(self):
        return False

    @property
    def require_x_translation(self):
        return True

177
178
179
180
    @property
    def require_y_translation(self):
        return True

181
182
183
184
185
186
187
188
189
190
191
    @property
    def require_z_translation(self):
        return True

    @property
    def has_diode(self):
        return False

    def is_different_sequence(self, entry):
        return True

192
193
194
195
196
197
    def register_step(
        self,
        url: DataUrl,
        entry_type: Optional[AcquisitionStep] = None,
        copy_frames=False,
    ) -> None:
198
199
        """

200
        :param DataUrl url: entry to be registered and contained in the
201
202
203
204
205
                                 acquisition
        :param entry_type: type of the entry if know. Overwise will be
                           'evaluated'
        """
        if entry_type is None:
206
            entry_type = get_entry_type(url=url, configuration=self.configuration)
207
208
209
        assert (
            entry_type is not AcquisitionStep.INITIALIZATION
        ), "Initialization are root node of a new sequence and not a scan of a sequence"
210

211
        if entry_type is None:
212
            _logger.warning("{} not recognized, skip it".format(url))
213
        else:
214
            self._registered_entries[url.path()] = entry_type
215
            self._copy_frames[url.path()] = copy_frames
216
217
            self._entries_o_path[url.path()] = url.data_path()
            # path from the original file. Haven't found another way to get it ?!
218

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
    def _get_valid_camera_names(self, instrument_grp: h5py.Group):
        # step one: try to get detector from nx property
        detectors = get_nx_detectors(instrument_grp)
        detectors = [grp.name.split("/")[-1] for grp in detectors]

        def filter_detectors(det_grps):
            if len(det_grps) > 0:
                _logger.info(
                    "{} detector found from NX_class attribute".format(len(det_grps))
                )
                if len(det_grps) > 1:
                    # if an option: pick the first one once orderered
                    # else ask user
                    if self._detector_sel_callback is None:
                        sel_det = det_grps[0]
                        _logger.warning(
                            "several detector found. Only one"
                            "is managed for now. Will pick {}"
                            "".format(sel_det)
                        )
                    else:
                        sel_det = self._detector_sel_callback(det_grps)
                        if sel_det is None:
                            _logger.warning("no detector given, avoid conversion")
                    det_grps = (sel_det,)
                return det_grps
            return None

        detectors = filter_detectors(det_grps=detectors)
        if detectors is not None:
            return detectors

        # step tow: get nx detector from shape...
        detectors = guess_nx_detector(instrument_grp)
        detectors = [grp.name.split("/")[-1] for grp in detectors]
        return filter_detectors(det_grps=detectors)

256
    def __get_data_from_camera(
257
        self,
258
259
        data_dataset: h5py.Dataset,
        data_name,
260
        frame_type,
261
        entry,
262
        entry_path,
263
        camera_dataset_url,
264
265
266
267
    ):
        if data_dataset.ndim == 2:
            shape = (1, data_dataset.shape[0], data_dataset.shape[1])
        elif data_dataset.ndim != 3:
268
269
270
            err = "dataset %s is expected to be 3D when %sD found." % (
                data_name,
                data_dataset.ndim,
271
            )
272
273
274
275
276
277
            if data_dataset.ndim == 1:
                err = "\n".join(
                    err,
                    "This might be a bliss-EDF dataset. Those are not handled by nxtomomill",
                )
            raise ValueError(err)
278
279
280
281
282
        else:
            shape = data_dataset.shape

        n_frame = shape[0]
        self._n_frames += n_frame
283
        self._n_frames_actual_bliss_scan = n_frame
284
285
286
287
288
289
290
291
292
293
294
295
296
297
        if self.dim_1 is None:
            self._dim_2 = shape[1]
            self._dim_1 = shape[2]
        else:
            if self._dim_1 != shape[2] or self._dim_2 != shape[1]:
                raise ValueError("Inconsistency in detector shapes")
        if self._data_type is None:
            self._data_type = data_dataset.dtype
        elif self._data_type != data_dataset.dtype:
            raise ValueError("detector frames have incoherent " "data types")

        # update image_key and image_key_control
        # Note: for now there is no image_key on the master file
        # should be added later.
298
        image_key_control = frame_type.to_image_key_control()
299
        self._image_key_control.extend([image_key_control.value] * n_frame)
300

301
302
303
304
305
        data_dataset_path = data_dataset.name.replace(entry.name, entry_path, 1)
        # replace data_dataset name by the original entry_path.
        # this is a workaround to use the dataset path on the
        # "treated file". Because .name if the name on the 'target'
        # file of the virtual dataset
306
        v_source = h5py.VirtualSource(
307
308
309
310
            camera_dataset_url.file_path(),
            data_dataset_path,
            data_dataset.shape,
            dtype=self._data_type,
311
        )
312
313
        self._virtual_sources.append(v_source)
        self._virtual_sources_len.append(n_frame)
314
315
        return n_frame

316
    def _treate_valid_camera(
317
318
319
320
321
322
323
        self,
        detector_node,
        entry,
        frame_type,
        input_file_path,
        entry_path,
        entry_url,
324
325
326
327
    ) -> bool:
        """
        return True if the entry contains frames
        """
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
        if "data_cast" in detector_node:
            _logger.warning(
                "!!! looks like this data has been cast. Take cast data for %s!!!"
                % detector_node
            )
            data_dataset = detector_node["data_cast"]
            data_name = "/".join((detector_node.name, "data_cast"))
        else:
            data_dataset = detector_node["data"]
            data_name = "/".join((detector_node.name, "data"))

        camera_dataset_url = DataUrl(
            file_path=entry_url.file_path(), data_path=data_name, scheme="silx"
        )

343
344
345
346
347
348
349
350
351
352
353
354
        n_frame = self.__get_data_from_camera(
            data_dataset,
            data_name=data_name,
            frame_type=frame_type,
            entry=entry,
            entry_path=entry_path,
            camera_dataset_url=camera_dataset_url,
        )
        # save information if this url must be embed / copy or not. Will be used later at nxtomo side
        self._copy_frames[camera_dataset_url.path()] = self._copy_frames[
            entry_url.path()
        ]
355

356
        # store rotation
357
358
        rots = self._get_rotation_angle(root_node=entry, n_frame=n_frame)[0]
        self._rotation_angle.extend(rots)
359

360
        if self.require_x_translation:
361
362
363
364
365
366
            self._x_translation.extend(
                self._get_x_translation(root_node=entry, n_frame=n_frame)[0]
            )
        else:
            self._x_translation = None

367
        if self.require_y_translation:
368
369
370
371
372
373
            self._y_translation.extend(
                self._get_y_translation(root_node=entry, n_frame=n_frame)[0]
            )
        else:
            self._y_translation = None

374
        if self.require_z_translation:
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
            self._z_translation.extend(
                self._get_z_translation(root_node=entry, n_frame=n_frame)[0]
            )
        else:
            self._z_translation = None

        # store acquisition time
        self._acq_expo_time.extend(
            self._get_expo_time(
                root_node=entry,
                detector_node=detector_node,
                n_frame=n_frame,
            )[0]
        )
        # retrieve data requested by plugins
        for resource_name in self._plugins_pos_resources:
391
            assert isinstance(resource_name, str), "resource_name should be a string"
392
393
394
395
            self._plugins_pos_resources[resource_name].extend(
                self._get_plugin_pos_resource(
                    root_node=entry,
                    resource_name=resource_name,
396
                    n_frame=None,
Henri Payno's avatar
Henri Payno committed
397
                )[0]
398
            )
399
400
401
402
403
404
        for resource_name in self._plugins_instr_resources:
            assert isinstance(resource_name, str), "resource_name should be a string"
            self._plugins_instr_resources[resource_name].extend(
                self._get_plugin_instr_resource(
                    root_node=entry,
                    resource_name=resource_name,
405
                    n_frame=None,
Henri Payno's avatar
Henri Payno committed
406
                )[0]
407
            )
408
        self._current_scan_n_frame = n_frame
409
410
411

    def camera_is_valid(self, det_name):
        assert isinstance(det_name, str)
412
413
        if self.configuration.valid_camera_names is None:
            return True
414
415
416
417
418
        for vcm in self.configuration.valid_camera_names:
            if fnmatch.fnmatch(det_name, vcm):
                return True
        return False

419
    def _preprocess_registered_entry(self, entry_url, type_):
420
421
422
423
424
425
426
427
428
429
430
        with EntryReader(entry_url) as entry:
            entry_path = self._entries_o_path[entry_url.path()]
            input_file_path = entry_url.file_path()
            input_file_path = os.path.abspath(
                os.path.relpath(input_file_path, os.getcwd())
            )
            input_file_path = os.path.realpath(input_file_path)
            if type_ is AcquisitionStep.INITIALIZATION:
                raise RuntimeError(
                    "no initialization should be registered."
                    "There should be only one per acquisition."
431
                )
432

433
434
            if "instrument" not in entry:
                _logger.error(
435
                    "no instrument group found in %s, unable to "
436
437
438
439
440
                    "retrieve frames" % entry.name
                )
                return

            # if we need to guess detector name(s)
441
            instrument_grp = entry["instrument"]
442
443
444
445
446
            if self.configuration.valid_camera_names is None:
                det_grps = self._get_valid_camera_names(instrument_grp)
                # update valid camera names
                self.configuration.valid_camera_names = det_grps

447
448
            has_frames = False
            for key, _ in instrument_grp.items():
449
450
451
452
453
454
                if (
                    "NX_class" in instrument_grp[key].attrs
                    and instrument_grp[key].attrs["NX_class"] == "NXdetector"
                ):
                    _logger.debug(
                        "Found one detector at %s for %s." % (key, entry.name)
455
                    )
456
457
458
459
460

                    # diode
                    if self.has_diode:
                        try:
                            diode_vals, diode_unit = self._get_diode(
461
                                root_node=entry, n_frame=self.n_frames
462
                            )
463
464
465
466
                        except Exception:
                            pass
                        else:
                            self._diode.extend(diode_vals)
467
                            self._diode_unit = diode_unit
468

469
                    if not self.camera_is_valid(key):
Henri Payno's avatar
Henri Payno committed
470
                        _logger.debug(f"ignore {key}, not a `valid` camera name")
471
472
                        continue
                    else:
473
                        detector_node = instrument_grp[key]
474
                        self._treate_valid_camera(
475
476
477
478
479
480
                            detector_node,
                            entry=entry,
                            frame_type=type_,
                            input_file_path=input_file_path,
                            entry_path=entry_path,
                            entry_url=entry_url,
481
                        )
482
483
484
485
486
487
488
489
490
491
492
493
                        has_frames = True
            # try to get some other metadata

            # handle frame time stamp
            start_time = self._get_start_time(entry)
            if start_time is not None:
                start_time = datetime.fromisoformat(start_time)
            end_time = self._get_end_time(entry)
            if end_time is not None:
                end_time = datetime.fromisoformat(end_time)

            if has_frames:
494
                self._register_frame_timestamp(entry, start_time, end_time)
495

496
497
            # handle electric current. Can retrieve some current even on bliss scan entry doesn;t containing directly frames
            self._register_machine_electric_current(entry, start_time, end_time)
498

499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
    def _register_machine_electric_current(
        self, entry: h5py.Group, start_time, end_time
    ):
        """Update machine electric current for provided entry (bliss scan"""
        (
            electric_currents,
            electric_current_unit,
        ) = self._get_electric_current(root_node=entry)
        electric_current_unit_ref = ElectricCurrentSystem.AMPERE
        # electric current will be saved as Ampere
        if electric_currents is not None and len(electric_currents) > 0:
            if electric_current_unit is None:
                electric_current_unit = ElectricCurrentSystem.MILLIAMPERE
                _logger.warning(
                    "No unit found for electric current. Consider it as mA."
                )

            unit_factor = (
                electric_current_unit_ref.value
                * ElectricCurrentSystem.from_str(electric_current_unit).value
            )

            new_know_electric_currents = {}
            if start_time is None or end_time is None:
                _logger.warning(
                    "Unable to find start_time and / or end_time. Will pick the first available electricl_current for the frame"
                )
                if start_time != end_time:
                    t_time = start_time or end_time
                    # if at least one can find out
529
530
531
                    new_know_electric_currents[
                        str_datetime_to_numpy_datetime64(t_time)
                    ] = (electric_currents[0] * unit_factor)
532
533
534
535
536
            elif len(electric_currents) == 1:
                # if we have only one value, consider the machine electric current is constant during this time
                # might be improved later if we can know if current is determine at the
                # beginning or the end. But should have no impact
                # as the time slot is short
537
538
539
                new_know_electric_currents[
                    str_datetime_to_numpy_datetime64(start_time)
                ] = (electric_currents[0] * unit_factor)
540
            else:
541
542
543
                # linspace from datetime within ms precision.
                # see https://localcoder.org/creating-numpy-linspace-out-of-datetime#credit_4
                # and https://stackoverflow.com/questions/37964100/creating-numpy-linspace-out-of-datetime
544
                timestamps = numpy.linspace(
545
546
                    start=str_datetime_to_numpy_datetime64(start_time).astype("f16"),
                    stop=str_datetime_to_numpy_datetime64(end_time).astype("f16"),
547
548
549
                    num=len(electric_currents),
                    endpoint=True,
                    dtype="<M8[ms]",
550
551
552
553
                )
                for timestamp, mach_electric_current in zip(
                    timestamps, electric_currents
                ):
554
                    new_know_electric_currents[timestamp.astype(numpy.datetime64)] = (
555
556
557
                        mach_electric_current * unit_factor
                    )
            self._known_machine_electric_current.update(new_know_electric_currents)
558

559
560
561
562
563
564
    def _register_frame_timestamp(self, entry: h5py.Group, start_time, end_time):
        """
        update frame time stamp for the provided entry (bliss scan)
        """
        if start_time is None or end_time is None:
            if start_time != end_time:
565
566
                t_time = str_datetime_to_numpy_datetime64(start_time or end_time)
                message = f"Unable to find start_time and / or end_time. Takes {t_time} as frame time stamp for {entry} "
567
                self._frames_timestamp.extend(
568
                    [t_time] * self._n_frames_actual_bliss_scan
569
570
571
572
573
574
                )
                _logger.warning(message)
            else:
                message = f"Unable to find start_time and end_time. Can't deduce frames time stamp for {entry}"
                _logger.error(message)
        else:
575
576
577
            frames_times_stamps_as_f8 = numpy.linspace(
                start=str_datetime_to_numpy_datetime64(start_time).astype("f8"),
                stop=str_datetime_to_numpy_datetime64(end_time).astype("f8"),
578
                num=self._n_frames_actual_bliss_scan,
579
580
                endpoint=True,
                dtype="<M8[ms]",
581
            )
582
583
584
585
            frames_times_stamps_as_f8 = [
                timestamp.astype("<M8[ms]") for timestamp in frames_times_stamps_as_f8
            ]
            self._frames_timestamp.extend(frames_times_stamps_as_f8)
586

587
    def _preprocess_registered_entries(self):
588
589
590
        """parse all frames of the different steps and retrieve data,
        image_key..."""
        self._n_frames = 0
591
        self._n_frames_actual_bliss_scan = 0
592
        # number of frame contains in X.1
593
594
595
596
597
598
599
600
        self._dim_1 = None
        self._dim_2 = None
        self._data_type = None
        self._x_translation = []
        self._y_translation = []
        self._z_translation = []
        self._image_key_control = []
        self._rotation_angle = []
Henri Payno's avatar
Henri Payno committed
601
        self._known_machine_electric_current = {}
602
        self._frames_timestamp = []
603
604
605
606
607
        self._virtual_sources = []
        self._virtual_sources_len = []
        self._diode = []
        self._acq_expo_time = []
        self._diode_unit = None
608
        self._copied_dataset = {}
609

610
611
612
613
614
615
616
617
        # if rotation motor is not defined try to deduce it from root_url/technique/scan/motor
        if self.configuration.rotation_angle_keys is None:
            rotation_motor = self._read_rotation_motor_name()
            if rotation_motor is not None:
                self.configuration.rotation_angle_keys = (rotation_motor,)
            else:
                self.configuration.rotation_angle_keys = tuple()

618
619
620
        # list of data virtual source for the virtual dataset
        for entry_url, type_ in self._registered_entries.items():
            url = DataUrl(path=entry_url)
621
            self._preprocess_registered_entry(url, type_)
622
623
624
625
626
627

        if len(self._diode) == 0:
            self._diode = None
        if self._diode is not None:
            self._diode = numpy.asarray(self._diode)
            self._diode = self._diode / self._diode.mean()
628
629
        for plugin in self._plugins:
            plugin.set_positioners_infos(self._plugins_pos_resources)
630
            plugin.set_instrument_infos(self._plugins_instr_resources)
631

payno's avatar
payno committed
632
    def _get_diode(self, root_node, n_frame) -> tuple:
633
        values, unit = self._get_node_values_for_frame_array(
payno's avatar
payno committed
634
            node=root_node["measurement"],
635
            n_frame=n_frame,
636
            keys=self.configuration.diode_keys,
637
638
639
640
641
            info_retrieve="diode",
            expected_unit="volt",
        )
        return values, unit

642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
    def get_already_defined_params(self, key):
        defined = self.__get_extra_param(key=key)
        if len(defined) > 1:
            raise ValueError("{} are aliases. Only one should be defined")
        elif len(defined) == 0:
            return None
        else:
            return list(defined.values())[0]

    def __get_extra_param(self, key) -> dict:
        """return already defined parameters for one key.
        A key as aliases so it returns a dict"""
        aliases = list(TomoHDF5Config.EXTRA_PARAMS_ALIASES[key])
        aliases.append(key)
        res = {}
        for alias in aliases:
            if alias in self.configuration.param_already_defined:
                res[alias] = self.configuration.param_already_defined[alias]
        return res

662
    def _generic_path_getter(self, path, message, level="warning", entry=None):
Henri Payno's avatar
Henri Payno committed
663
664
        """
        :param str level: level can be logging.level values : "warning", "error", "info"
665
        :param H5group entry: user can provide directly an entry to be used as an open h5Group
Henri Payno's avatar
Henri Payno committed
666
        """
667
668
        if self.root_url is None:
            return None
669
        self._check_has_metadata()
670
671
672
673

        def process(h5_group):
            if path in h5_group:
                return h5py_read_dataset(h5_group[path])
674
            else:
Henri Payno's avatar
Henri Payno committed
675
676
                if message is not None:
                    getattr(_logger, level)(message)
677
                return None
678

679
680
681
682
683
684
        if entry is None:
            with self.read_entry() as h5_group:
                return process(h5_group)
        else:
            return process(entry)

Henri Payno's avatar
Henri Payno committed
685
686
687
688
689
690
691
692
693
694
695
696
    def _get_source_name(self):
        """ """
        return self._generic_path_getter(
            path=self._SOURCE_NAME, message="Unable to find source name", level="info"
        )

    def _get_source_type(self):
        """ """
        return self._generic_path_getter(
            path=self._SOURCE_TYPE, message="Unable to find source type", level="info"
        )

payno's avatar
payno committed
697
698
699
    def _get_title(self):
        """return acquisition title"""
        return self._generic_path_getter(
Henri Payno's avatar
Henri Payno committed
700
            path=self._TITLE_PATH, message="Unable to find title"
payno's avatar
payno committed
701
702
        )

payno's avatar
payno committed
703
704
705
    def _get_instrument_name(self):
        """:return instrument title / name"""
        return self._generic_path_getter(
Henri Payno's avatar
Henri Payno committed
706
707
708
            path=self._INSTRUMENT_NAME_PATH,
            message="Unable to find instrument name",
            level="info",
payno's avatar
payno committed
709
710
        )

711
    def _get_dataset_name(self):
712
        """return name of the acquisition"""
713
        return self._generic_path_getter(
714
            path=self._DATASET_NAME_PATH,
Henri Payno's avatar
Henri Payno committed
715
            message="No name describing the acquisition has been "
716
717
718
719
720
721
722
            "found, Name dataset will be skip",
        )

    def _get_sample_name(self):
        """return sample name"""
        return self._generic_path_getter(
            path=self._SAMPLE_NAME_PATH,
Henri Payno's avatar
Henri Payno committed
723
            message="No sample name has been "
724
725
            "found, Sample name dataset will be skip",
        )
726
727
728
729

    def _get_grp_size(self):
        """return the nb_scans composing the zseries if is part of a group
        of sequence"""
730
731
        return self._generic_path_getter(
            path=self._GRP_SIZE_PATH,
Henri Payno's avatar
Henri Payno committed
732
            message=None,
733
734
735
736
737
        )

    def _get_tomo_n(self):
        return self._generic_path_getter(
            path=self._TOMO_N_PATH,
Henri Payno's avatar
Henri Payno committed
738
            message="unable to find information regarding tomo_n",
739
740
        )

741
    def _get_start_time(self, entry=None):
742
        return self._generic_path_getter(
Henri Payno's avatar
Henri Payno committed
743
744
745
            path=self._START_TIME_PATH,
            message="Unable to find start time",
            level="info",
746
            entry=entry,
747
748
        )

749
    def _get_end_time(self, entry=None):
Henri Payno's avatar
Henri Payno committed
750
        return self._generic_path_getter(
751
752
753
754
            path=self._END_TIME_PATH,
            message="Unable to find end time",
            level="info",
            entry=entry,
Henri Payno's avatar
Henri Payno committed
755
        )
756

757
758
    def _get_energy(self, ask_if_0, input_callback):
        """return tuple(energy, unit)"""
759
760
        if self.root_url is None:
            return None, None
761
        self._check_has_metadata()
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
        with self.read_entry() as entry:
            if self._ENERGY_PATH in entry:
                energy = h5py_read_dataset(entry[self._ENERGY_PATH])
                unit = self._get_unit(entry[self._ENERGY_PATH], default_unit="kev")
                if energy == 0 and ask_if_0:
                    desc = (
                        "Energy has not been registered. Please enter "
                        "incoming beam energy (in kev):"
                    )
                    if input_callback is None:
                        en = input(desc)
                    else:
                        en = input_callback("energy", desc)
                    if energy is not None:
                        energy = float(en)
                return energy, unit
778
            else:
779
780
781
782
783
784
785
                mess = "unable to find energy for entry {}.".format(entry)
                if self.raise_error_if_issue:
                    raise ValueError(mess)
                else:
                    mess += " Default value will be set (19kev)"
                    _logger.warning(mess)
                    return 19.0, "kev"
786
787
788

    def _get_distance(self):
        """return tuple(distance, unit)"""
789
790
        if self.root_url is None:
            return None, None
791
        self._check_has_metadata()
792
        with self.read_entry() as entry:
793
794
795
796
797
798
            for key in self.configuration.sample_detector_distance_paths:
                if key in entry:
                    node = entry[key]
                    distance = h5py_read_dataset(node)
                    unit = self._get_unit(node, default_unit="cm")
                    # convert to meter
payno's avatar
payno committed
799
800
801
                    distance = (
                        distance * metricsystem.MetricSystem.from_value(unit).value
                    )
802
803
804
805
                    return distance, "m"
            mess = "unable to find distance for entry {}.".format(entry)
            if self.raise_error_if_issue:
                raise ValueError(mess)
806
            else:
807
808
809
                mess += "Default value will be set (1m)"
                _logger.warning(mess)
                return 1.0, "m"
810
811
812

    def _get_pixel_size(self, axis):
        """return tuple(pixel_size, unit)"""
813
814
        if self.root_url is None:
            return None, None
815
816
817
        assert axis in ("x", "y")
        self._check_has_metadata()
        keys = (
818
            self.configuration.x_pixel_size_paths
819
            if axis == "x"
820
            else self.configuration.y_pixel_size_paths
821
        )
822
823
824
825
826
827
828
829
830
831
832
833
        with self.read_entry() as entry:
            for key in keys:
                if key in entry:
                    node = entry[key]
                    node_item = h5py_read_dataset(node)
                    # if the pixel size is provided as x, y
                    if isinstance(node_item, numpy.ndarray):
                        if len(node_item) > 1 and axis == "y":
                            size_ = node_item[1]
                        else:
                            size_ = node_item[0]
                    # if this is a single value
834
                    else:
835
836
837
838
839
                        size_ = node_item
                    unit = self._get_unit(node, default_unit="micrometer")
                    # convert to meter
                    size_ = size_ * metricsystem.MetricSystem.from_value(unit).value
                    return size_, "m"
840

841
842
843
844
845
846
847
            mess = "unable to find {} pixel size for entry {}".format(axis, entry)
            if self.raise_error_if_issue:
                raise ValueError(mess)
            else:
                mess += "default value will be set to 10-6m"
                _logger.warning(mess)
                return 10e-6, "m"
848
849

    def _get_field_of_fiew(self):
850
851
        if self.configuration.field_of_view is not None:
            return self.configuration.field_of_view.value
852
853
        if self.root_url is None:
            return None
854
855
856
857
858
859
860
861
862
863
864
865
        with self.read_entry() as entry:
            if self._FOV_PATH in entry:
                return h5py_read_dataset(entry[self._FOV_PATH])
            else:
                mess = (
                    "unable to find information regarding field of view for"
                    " entry {}".format(entry)
                )
                # FOV is optional: shouldn't won't raise an error
                mess += "set it to default value (Full)"
                _logger.warning(mess)
                return "Full"
866
867
868

    def _get_estimated_cor_from_motor(self, pixel_size):
        """given pixel is expected to be given in meter"""
869
870
        if self.root_url is None:
            return None, None
871
872
873
874
875
876
877
878
        with self.read_entry() as entry:
            if self.configuration.y_rot_key in entry:
                y_rot = h5py_read_dataset(entry[self.configuration.y_rot_key])
            else:
                _logger.warning(
                    "unable to find information on positioner {}".format(
                        self.configuration.y_rot_key
                    )
879
                )
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
                return None, None
            # y_rot is provided in mm when pixel size is in meter.
            y_rot = y_rot * metricsystem.millimeter.value

            if pixel_size is None:
                mess = (
                    "pixel size is required to estimate the cor from the "
                    "motor position on pixels"
                )
                if self.raise_error_if_issue:
                    raise ValueError(mess)
                else:
                    mess += " Set default value (0m)"
                    _logger.warning(mess)
                    return 0, "m"
895
            else:
896
                return y_rot / pixel_size, "pixels"
897

898
899
900
    def to_NXtomos(self, request_input, input_callback, check_tomo_n=True) -> tuple:
        self._preprocess_registered_entries()

901
        nx_tomo = NXtomo()
902
903
904
905
906
907
908
909

        # 1. root level information
        # start and end time
        nx_tomo.start_time = self._get_start_time()
        nx_tomo.end_time = self._get_end_time()

        # title
        nx_tomo.title = self._get_dataset_name()
910
911
        # group size
        nx_tomo.group_size = self._get_grp_size()
912
913
914
915
916
917
918
919

        # 2. define beam
        energy, unit = self._get_user_settable_parameter(
            param_key=TomoHDF5Config.EXTRA_PARAMS_ENERGY_DK,
            fallback_fct=self._get_energy,
            dtype=float,
            input_callback=input_callback,
            ask_if_0=request_input,
Henri Payno's avatar
Henri Payno committed
920
        )
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
        if energy is not None:
            # TODO: better manamgent of energy ? might be energy.beam or energy.instrument.beam ?
            nx_tomo.energy = energy
            nx_tomo.energy.unit = unit

        # 3. define instrument
        nx_tomo.instrument.detector.data = self._virtual_sources
        nx_tomo.instrument.detector.image_key_control = self.image_key_control
        nx_tomo.instrument.detector.count_time = self._acq_expo_time
        nx_tomo.instrument.detector.count_time.unit = "s"
        # distance
        distance, unit = self._get_user_settable_parameter(
            param_key=TomoHDF5Config.EXTRA_PARAMS_DISTANCE,
            fallback_fct=self._get_distance,
            dtype=float,
        )
        if distance is not None:
            nx_tomo.instrument.detector.distance = distance
            if nx_tomo.instrument.detector.distance is not None:
                nx_tomo.instrument.detector.distance.unit = unit
        # x and y pixel size
        x_pixel_size, unit = self._get_user_settable_parameter(
            param_key=TomoHDF5Config.EXTRA_PARAMS_X_PIXEL_SIZE_DK,
            fallback_fct=self._get_pixel_size,
            dtype=float,
            axis="x",
        )
        nx_tomo.instrument.detector.x_pixel_size = x_pixel_size
        if unit is not None:
            nx_tomo.instrument.detector.x_pixel_size.unit = unit

        y_pixel_size, unit = self._get_user_settable_parameter(
            param_key=TomoHDF5Config.EXTRA_PARAMS_Y_PIXEL_SIZE_DK,
            fallback_fct=self._get_pixel_size,
            dtype=float,
            axis="y",
        )
        nx_tomo.instrument.detector.y_pixel_size = y_pixel_size
        if unit is not None:
            nx_tomo.instrument.detector.y_pixel_size.unit = unit

        fov = self._get_field_of_fiew()
        if fov is not None:
964
            nx_tomo.instrument.detector.field_of_view = fov
965
966
967
            if fov.lower() == "half":
                estimated_cor, unit = self._get_estimated_cor_from_motor(
                    pixel_size=y_pixel_size
Henri Payno's avatar
Henri Payno committed
968
                )
969
970
971
972
973
                if estimated_cor is not None:
                    nx_tomo.instrument.detector.estimated_cor_from_motor = estimated_cor

        # define tomo_n
        nx_tomo.instrument.detector.tomo_n = self._get_tomo_n()
Henri Payno's avatar
Henri Payno committed
974

975
        # 4. define nx source
976
977
978
979
980
981
982
983
984
985
986
        source_name = self._get_source_name()
        nx_tomo.instrument.source.name = source_name
        source_type = self._get_source_type()
        if source_type is not None:
            if "synchrotron" in source_type.lower():
                source_type = SourceType.SYNCHROTRON_X_RAY_SOURCE.value
            # drop a warning if the source type is invalid
            if source_type not in SourceType.values():
                _logger.warning(
                    "Source type ({}) is not a 'standard value'".format(source_type)
                )
Henri Payno's avatar
Henri Payno committed
987

988
989
        nx_tomo.instrument.source.type = source_type

990
        # 5. define sample
991
992
993
994
995
996
997
998
999
        nx_tomo.sample.name = self._get_sample_name()
        nx_tomo.sample.rotation_angle = self.rotation_angle
        nx_tomo.sample.x_translation.value = self.x_translation
        nx_tomo.sample.x_translation.unit = "m"
        nx_tomo.sample.y_translation.value = self.y_translation
        nx_tomo.sample.y_translation.unit = "m"
        nx_tomo.sample.z_translation.value = self.z_translation
        nx_tomo.sample.z_translation.unit = "m"

1000
        # 6. define control
For faster browsing, not all history is shown. View entire blame