Commit 1544c6e0 authored by payno's avatar payno
Browse files

[hdf5scan] replace usage of h5py.File by HDF5File

parent 53644930
Pipeline #31838 passed with stages
in 5 minutes and 29 seconds
...@@ -39,6 +39,7 @@ import numpy ...@@ -39,6 +39,7 @@ import numpy
from silx.io.url import DataUrl from silx.io.url import DataUrl
from silx.utils.enum import Enum as _Enum from silx.utils.enum import Enum as _Enum
from tomoscan.utils import docstring from tomoscan.utils import docstring
from tomoscan.io import HDF5File
from silx.io.utils import get_data from silx.io.utils import get_data
from ..unitsystem import metricsystem from ..unitsystem import metricsystem
from .utils import get_compacted_dataslices from .utils import get_compacted_dataslices
...@@ -231,8 +232,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -231,8 +232,7 @@ class HDF5TomoScan(TomoScanBase):
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
raise ValueError('given file path should be a file') raise ValueError('given file path should be a file')
with h5py.File(file_path, 'r', swmr=True) as h5f: with HDF5File(file_path, 'r', swmr=True) as h5f:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
for root_node in h5f.keys(): for root_node in h5f.keys():
node = h5f[root_node] node = h5f[root_node]
if HDF5TomoScan.node_is_nxtomo(node) is True: if HDF5TomoScan.node_is_nxtomo(node) is True:
...@@ -406,8 +406,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -406,8 +406,7 @@ class HDF5TomoScan(TomoScanBase):
def rotation_angle(self) -> typing.Union[None, list]: def rotation_angle(self) -> typing.Union[None, list]:
if self._rotation_angles is None: if self._rotation_angles is None:
self._check_hdf5scan_validity() self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
_rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()] _rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()]
# cast in float # cast in float
self._rotation_angles = tuple([float(angle) for angle in _rotation_angles]) self._rotation_angles = tuple([float(angle) for angle in _rotation_angles])
...@@ -417,8 +416,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -417,8 +416,7 @@ class HDF5TomoScan(TomoScanBase):
def image_key(self) -> typing.Union[list, None]: def image_key(self) -> typing.Union[list, None]:
if self._entry and self._image_keys is None: if self._entry and self._image_keys is None:
self._check_hdf5scan_validity() self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()] self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()]
return self._image_keys return self._image_keys
...@@ -426,8 +424,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -426,8 +424,7 @@ class HDF5TomoScan(TomoScanBase):
def image_key_control(self) -> typing.Union[list, None]: def image_key_control(self) -> typing.Union[list, None]:
if self._entry and self._image_keys_control is None: if self._entry and self._image_keys_control is None:
self._check_hdf5scan_validity() self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]: if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]:
self._image_keys_control = h5_file[self._entry][self._IMG_KEY_CONTROL_PATH][()] self._image_keys_control = h5_file[self._entry][self._IMG_KEY_CONTROL_PATH][()]
else: else:
...@@ -500,8 +497,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -500,8 +497,7 @@ class HDF5TomoScan(TomoScanBase):
def _get_x_y_pixel_values(self): def _get_x_y_pixel_values(self):
"""read x and y pixel values""" """read x and y pixel values"""
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
x_pixel_dataset = h5_file[self._entry][self._X_PIXEL_SIZE_PATH] x_pixel_dataset = h5_file[self._entry][self._X_PIXEL_SIZE_PATH]
_x_pixel_size = self._get_value(x_pixel_dataset, default_unit='meter') _x_pixel_size = self._get_value(x_pixel_dataset, default_unit='meter')
y_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_SIZE_PATH] y_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_SIZE_PATH]
...@@ -509,8 +505,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -509,8 +505,7 @@ class HDF5TomoScan(TomoScanBase):
return _x_pixel_size, _y_pixel_size return _x_pixel_size, _y_pixel_size
def _get_x_y_magnified_pixel_values(self): def _get_x_y_magnified_pixel_values(self):
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
x_m_pixel_dataset = h5_file[self._entry][self._X_PIXEL_MAG_SIZE_PATH] x_m_pixel_dataset = h5_file[self._entry][self._X_PIXEL_MAG_SIZE_PATH]
_x_m_pixel_size = self._get_value(x_m_pixel_dataset, default_unit='meter') _x_m_pixel_size = self._get_value(x_m_pixel_dataset, default_unit='meter')
y_m_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_MAG_SIZE_PATH] y_m_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_MAG_SIZE_PATH]
...@@ -518,8 +513,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -518,8 +513,7 @@ class HDF5TomoScan(TomoScanBase):
return _x_m_pixel_size, _y_m_pixel_size return _x_m_pixel_size, _y_m_pixel_size
def _get_fov(self): def _get_fov(self):
with h5py.File(self.master_file, 'r', swmr=True, libver='latest') as h5_file: with HDF5File(self.master_file, 'r', swmr=True, libver='latest') as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
if self._FOV_PATH in h5_file[self._entry]: if self._FOV_PATH in h5_file[self._entry]:
fov = h5_file[self._entry][self._FOV_PATH][()] fov = h5_file[self._entry][self._FOV_PATH][()]
return _FOV.from_value(fov) return _FOV.from_value(fov)
...@@ -562,8 +556,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -562,8 +556,7 @@ class HDF5TomoScan(TomoScanBase):
if (self._distance is None and self.master_file and if (self._distance is None and self.master_file and
os.path.exists(self.master_file)): os.path.exists(self.master_file)):
self._check_hdf5scan_validity() self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
distance_dataset = h5_file[self._entry][self._DISTANCE_PATH] distance_dataset = h5_file[self._entry][self._DISTANCE_PATH]
self._distance = self._get_value(distance_dataset, default_unit='m') self._distance = self._get_value(distance_dataset, default_unit='m')
return self._distance return self._distance
...@@ -584,8 +577,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -584,8 +577,7 @@ class HDF5TomoScan(TomoScanBase):
if (self._energy is None and self.master_file and if (self._energy is None and self.master_file and
os.path.exists(self.master_file)): os.path.exists(self.master_file)):
self._check_hdf5scan_validity() self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
energy_dataset = h5_file[self._entry][self._ENERGY_PATH] energy_dataset = h5_file[self._entry][self._ENERGY_PATH]
self._energy = self._get_value(energy_dataset, default_unit='keV') self._energy = self._get_value(energy_dataset, default_unit='keV')
return self._energy return self._energy
...@@ -693,8 +685,7 @@ class HDF5TomoScan(TomoScanBase): ...@@ -693,8 +685,7 @@ class HDF5TomoScan(TomoScanBase):
raise ValueError('No master file provided') raise ValueError('No master file provided')
if self.entry is None: if self.entry is None:
raise ValueError('No entry provided') raise ValueError('No entry provided')
with h5py.File(self.master_file, 'r', swmr=True) as h5_file: with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
if self._entry not in h5_file: if self._entry not in h5_file:
raise ValueError('Given entry %s is not in the master ' raise ValueError('Given entry %s is not in the master '
'file %s' % (self._entry, self.master_file)) 'file %s' % (self._entry, self.master_file))
......
# -*- coding: utf-8 -*-
#
# This file is part of the nexus writer service of the BLISS project.
#
# Code is maintained by the ESRF Data Analysis Unit.
#
# Original author: Wout de Nolf
#
# Copyright (c) 2015-2020 ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.
import h5py
from tomoscan.utils import SharedLockPool
import os
from contextlib import contextmanager
import traceback
import errno
HASSWMR = h5py.version.hdf5_version_tuple >= h5py.get_config().swmr_min_hdf5_version
class HDF5File(h5py.File):
"""File to secure reading and writing within h5py
code originally from bliss.nexus_writer_service.io.nexus
"""
_LOCKPOOL = SharedLockPool()
def __init__(
self, filename, mode, enable_file_locking=None, swmr=None, **kwargs
):
"""
:param str filename:
:param str mode:
:param bool enable_file_locking: by default it is disabled for `mode=='r'`
and enabled in all other modes
:param bool swmr: when not specified: try both modes when `mode=='r'`
:param **kwargs: see `h5py.File.__init__`
"""
if mode not in ('r', 'r+', 'w', 'w-', 'x', 'a'):
raise ValueError('invalid mode {}'.format(mode))
with self._protect_init(filename):
# https://support.hdfgroup.org/HDF5/docNewFeatures/SWMR/Design-HDF5-FileLocking.pdf
if not HASSWMR and swmr:
swmr = False
libver = kwargs.get("libver")
if swmr:
kwargs["libver"] = "latest"
if enable_file_locking is None:
enable_file_locking = mode != "r"
if enable_file_locking:
os.environ["HDF5_USE_FILE_LOCKING"] = "TRUE"
else:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
kwargs["track_order"] = True
try:
super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
if mode != "r" and swmr:
# Try setting writing in SWMR mode
try:
self.swmr_mode = True
except Exception:
pass
except OSError as e:
if (
swmr is not None
or mode != "r"
or not HASSWMR
or not isErrno(e, errno.EAGAIN)
):
raise
# Try reading with opposite SWMR mode
swmr = not swmr
if swmr:
kwargs["libver"] = "latest"
else:
kwargs["libver"] = libver
super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
@contextmanager
def _protect_init(self, filename):
"""Makes sure no other file is opened/created
or protected sections associated to the filename
are executed.
"""
lockname = os.path.abspath(filename)
with self._LOCKPOOL.acquire(None):
with self._LOCKPOOL.acquire(lockname):
yield
@contextmanager
def protect(self):
"""Protected section associated to this file.
"""
lockname = os.path.abspath(self.filename)
with self._LOCKPOOL.acquire(lockname):
yield
def isErrno(e, errno):
"""
:param OSError e:
:returns bool:
"""
# Because e.__cause__ is None for chained exceptions
return "errno = {}".format(errno) in "".join(traceback.format_exc())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment