[hdf5scan] replace usage of h5py.File by HDF5File

tomoscan/esrf/hdf5scan.py

@@ -39,6 +39,7 @@ import numpy
 from silx.io.url import DataUrl
 from silx.utils.enum import Enum as _Enum
 from tomoscan.utils import docstring
+from tomoscan.io import HDF5File
 from silx.io.utils import get_data
 from ..unitsystem import metricsystem
 from .utils import get_compacted_dataslices
@@ -231,8 +232,7 @@ class HDF5TomoScan(TomoScanBase):
         if not os.path.isfile(file_path):
             raise ValueError('given file path should be a file')
 
-        with h5py.File(file_path, 'r', swmr=True) as h5f:
-            os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+        with HDF5File(file_path, 'r', swmr=True) as h5f:
             for root_node in h5f.keys():
                 node = h5f[root_node]
                 if HDF5TomoScan.node_is_nxtomo(node) is True:
@@ -406,8 +406,7 @@ class HDF5TomoScan(TomoScanBase):
     def rotation_angle(self) -> typing.Union[None, list]:
         if self._rotation_angles is None:
             self._check_hdf5scan_validity()
-            with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
                 _rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()]
                 # cast in float
                 self._rotation_angles = tuple([float(angle) for angle in _rotation_angles])
@@ -417,8 +416,7 @@ class HDF5TomoScan(TomoScanBase):
     def image_key(self) -> typing.Union[list, None]:
         if self._entry and self._image_keys is None:
             self._check_hdf5scan_validity()
-            with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
                 self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()]
         return self._image_keys
 
@@ -426,8 +424,7 @@ class HDF5TomoScan(TomoScanBase):
     def image_key_control(self) -> typing.Union[list, None]:
         if self._entry and self._image_keys_control is None:
             self._check_hdf5scan_validity()
-            with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
                 if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]:
                     self._image_keys_control = h5_file[self._entry][self._IMG_KEY_CONTROL_PATH][()]
                 else:
@@ -500,8 +497,7 @@ class HDF5TomoScan(TomoScanBase):
 
     def _get_x_y_pixel_values(self):
         """read x and y pixel values"""
-        with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-            os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+        with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
             x_pixel_dataset = h5_file[self._entry][self._X_PIXEL_SIZE_PATH]
             _x_pixel_size = self._get_value(x_pixel_dataset, default_unit='meter')
             y_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_SIZE_PATH]
@@ -509,8 +505,7 @@ class HDF5TomoScan(TomoScanBase):
         return _x_pixel_size, _y_pixel_size
 
     def _get_x_y_magnified_pixel_values(self):
-        with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-            os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+        with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
             x_m_pixel_dataset = h5_file[self._entry][self._X_PIXEL_MAG_SIZE_PATH]
             _x_m_pixel_size = self._get_value(x_m_pixel_dataset, default_unit='meter')
             y_m_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_MAG_SIZE_PATH]
@@ -518,8 +513,7 @@ class HDF5TomoScan(TomoScanBase):
         return _x_m_pixel_size, _y_m_pixel_size
 
     def _get_fov(self):
-        with h5py.File(self.master_file, 'r', swmr=True, libver='latest') as h5_file:
-            os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+        with HDF5File(self.master_file, 'r', swmr=True, libver='latest') as h5_file:
             if self._FOV_PATH in h5_file[self._entry]:
                 fov = h5_file[self._entry][self._FOV_PATH][()]
                 return _FOV.from_value(fov)
@@ -562,8 +556,7 @@ class HDF5TomoScan(TomoScanBase):
         if (self._distance is None and self.master_file and
                 os.path.exists(self.master_file)):
             self._check_hdf5scan_validity()
-            with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
                 distance_dataset = h5_file[self._entry][self._DISTANCE_PATH]
                 self._distance = self._get_value(distance_dataset, default_unit='m')
         return self._distance
@@ -584,8 +577,7 @@ class HDF5TomoScan(TomoScanBase):
         if (self._energy is None and self.master_file and
                 os.path.exists(self.master_file)):
             self._check_hdf5scan_validity()
-            with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
                 energy_dataset = h5_file[self._entry][self._ENERGY_PATH]
                 self._energy = self._get_value(energy_dataset, default_unit='keV')
         return self._energy
@@ -693,8 +685,7 @@ class HDF5TomoScan(TomoScanBase):
             raise ValueError('No master file provided')
         if self.entry is None:
             raise ValueError('No entry provided')
-        with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
-            os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+        with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
             if self._entry not in h5_file:
                 raise ValueError('Given entry %s is not in the master '
                                  'file %s' % (self._entry, self.master_file))

tomoscan/io.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the nexus writer service of the BLISS project.
+#
+# Code is maintained by the ESRF Data Analysis Unit.
+#
+# Original author: Wout de Nolf
+#
+# Copyright (c) 2015-2020 ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+import h5py
+from tomoscan.utils import SharedLockPool
+import os
+from contextlib import contextmanager
+import traceback
+import errno
+
+HASSWMR = h5py.version.hdf5_version_tuple >= h5py.get_config().swmr_min_hdf5_version
+
+
+class HDF5File(h5py.File):
+    """File to secure reading and writing within h5py
+
+    code originally from bliss.nexus_writer_service.io.nexus
+    """
+    _LOCKPOOL = SharedLockPool()
+
+    def __init__(
+        self, filename, mode, enable_file_locking=None, swmr=None, **kwargs
+    ):
+        """
+        :param str filename:
+        :param str mode:
+        :param bool enable_file_locking: by default it is disabled for `mode=='r'`
+                                         and enabled in all other modes
+        :param bool swmr: when not specified: try both modes when `mode=='r'`
+        :param **kwargs: see `h5py.File.__init__`
+        """
+        if mode not in ('r', 'r+', 'w', 'w-', 'x', 'a'):
+            raise ValueError('invalid mode {}'.format(mode))
+
+        with self._protect_init(filename):
+            # https://support.hdfgroup.org/HDF5/docNewFeatures/SWMR/Design-HDF5-FileLocking.pdf
+            if not HASSWMR and swmr:
+                swmr = False
+            libver = kwargs.get("libver")
+            if swmr:
+                kwargs["libver"] = "latest"
+            if enable_file_locking is None:
+                enable_file_locking = mode != "r"
+            if enable_file_locking:
+                os.environ["HDF5_USE_FILE_LOCKING"] = "TRUE"
+            else:
+                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
+            kwargs["track_order"] = True
+            try:
+                super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
+                if mode != "r" and swmr:
+                    # Try setting writing in SWMR mode
+                    try:
+                        self.swmr_mode = True
+                    except Exception:
+                        pass
+            except OSError as e:
+                if (
+                    swmr is not None
+                    or mode != "r"
+                    or not HASSWMR
+                    or not isErrno(e, errno.EAGAIN)
+                ):
+                    raise
+                # Try reading with opposite SWMR mode
+                swmr = not swmr
+                if swmr:
+                    kwargs["libver"] = "latest"
+                else:
+                    kwargs["libver"] = libver
+                super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
+
+    @contextmanager
+    def _protect_init(self, filename):
+        """Makes sure no other file is opened/created
+        or protected sections associated to the filename
+        are executed.
+        """
+        lockname = os.path.abspath(filename)
+        with self._LOCKPOOL.acquire(None):
+            with self._LOCKPOOL.acquire(lockname):
+                yield
+
+    @contextmanager
+    def protect(self):
+        """Protected section associated to this file.
+        """
+        lockname = os.path.abspath(self.filename)
+        with self._LOCKPOOL.acquire(lockname):
+            yield
+
+
+def isErrno(e, errno):
+    """
+    :param OSError e:
+    :returns bool:
+    """
+    # Because e.__cause__ is None for chained exceptions
+    return "errno = {}".format(errno) in "".join(traceback.format_exc())