Commit 1cae8bf0 authored by Pierre Paleo's avatar Pierre Paleo
Browse files

Merge branch 'hdf5_scan_file_lock' into '0.3'

set `HDF5_USE_FILE_LOCKING` when read a file

See merge request !23
parents 79e6de9d 46336dfb
Pipeline #31950 passed with stages
in 5 minutes and 30 seconds
......@@ -39,6 +39,7 @@ import numpy
from silx.io.url import DataUrl
from silx.utils.enum import Enum as _Enum
from tomoscan.utils import docstring
from tomoscan.io import HDF5File
from silx.io.utils import get_data
from ..unitsystem import metricsystem
from .utils import get_compacted_dataslices
......@@ -231,7 +232,7 @@ class HDF5TomoScan(TomoScanBase):
if not os.path.isfile(file_path):
raise ValueError('given file path should be a file')
with h5py.File(file_path, 'r', swmr=True) as h5f:
with HDF5File(file_path, 'r', swmr=True) as h5f:
for root_node in h5f.keys():
node = h5f[root_node]
if HDF5TomoScan.node_is_nxtomo(node) is True:
......@@ -405,7 +406,7 @@ class HDF5TomoScan(TomoScanBase):
def rotation_angle(self) -> typing.Union[None, list]:
if self._rotation_angles is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
_rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()]
# cast in float
self._rotation_angles = tuple([float(angle) for angle in _rotation_angles])
......@@ -415,7 +416,7 @@ class HDF5TomoScan(TomoScanBase):
def image_key(self) -> typing.Union[list, None]:
if self._entry and self._image_keys is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()]
return self._image_keys
......@@ -423,7 +424,7 @@ class HDF5TomoScan(TomoScanBase):
def image_key_control(self) -> typing.Union[list, None]:
if self._entry and self._image_keys_control is None:
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]:
self._image_keys_control = h5_file[self._entry][self._IMG_KEY_CONTROL_PATH][()]
else:
......@@ -496,7 +497,7 @@ class HDF5TomoScan(TomoScanBase):
def _get_x_y_pixel_values(self):
"""read x and y pixel values"""
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
x_pixel_dataset = h5_file[self._entry][self._X_PIXEL_SIZE_PATH]
_x_pixel_size = self._get_value(x_pixel_dataset, default_unit='meter')
y_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_SIZE_PATH]
......@@ -504,7 +505,7 @@ class HDF5TomoScan(TomoScanBase):
return _x_pixel_size, _y_pixel_size
def _get_x_y_magnified_pixel_values(self):
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
x_m_pixel_dataset = h5_file[self._entry][self._X_PIXEL_MAG_SIZE_PATH]
_x_m_pixel_size = self._get_value(x_m_pixel_dataset, default_unit='meter')
y_m_pixel_dataset = h5_file[self._entry][self._Y_PIXEL_MAG_SIZE_PATH]
......@@ -512,7 +513,7 @@ class HDF5TomoScan(TomoScanBase):
return _x_m_pixel_size, _y_m_pixel_size
def _get_fov(self):
with h5py.File(self.master_file, 'r', swmr=True, libver='latest') as h5_file:
with HDF5File(self.master_file, 'r', swmr=True, libver='latest') as h5_file:
if self._FOV_PATH in h5_file[self._entry]:
fov = h5_file[self._entry][self._FOV_PATH][()]
return _FOV.from_value(fov)
......@@ -555,7 +556,7 @@ class HDF5TomoScan(TomoScanBase):
if (self._distance is None and self.master_file and
os.path.exists(self.master_file)):
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
distance_dataset = h5_file[self._entry][self._DISTANCE_PATH]
self._distance = self._get_value(distance_dataset, default_unit='m')
return self._distance
......@@ -576,7 +577,7 @@ class HDF5TomoScan(TomoScanBase):
if (self._energy is None and self.master_file and
os.path.exists(self.master_file)):
self._check_hdf5scan_validity()
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
energy_dataset = h5_file[self._entry][self._ENERGY_PATH]
self._energy = self._get_value(energy_dataset, default_unit='keV')
return self._energy
......@@ -684,7 +685,7 @@ class HDF5TomoScan(TomoScanBase):
raise ValueError('No master file provided')
if self.entry is None:
raise ValueError('No entry provided')
with h5py.File(self.master_file, 'r', swmr=True) as h5_file:
with HDF5File(self.master_file, 'r', swmr=True) as h5_file:
if self._entry not in h5_file:
raise ValueError('Given entry %s is not in the master '
'file %s' % (self._entry, self.master_file))
......
......@@ -58,9 +58,9 @@ class TestTomoBaseHashable(unittest.TestCase):
def test_is_hashable(self):
tomo_base = TomoScanBase(scan=self._folder, type_='toto')
self.assertTrue(isinstance(tomo_base, collections.Hashable))
self.assertTrue(isinstance(tomo_base, collections.abc.Hashable))
tomo_scan = EDFTomoScan(self._folder)
self.assertTrue(isinstance(tomo_scan, collections.Hashable))
self.assertTrue(isinstance(tomo_scan, collections.abc.Hashable))
class TestScanFactory(unittest.TestCase):
......
......@@ -81,7 +81,7 @@ class TestHDF5Scan(HDF5TestBaseClass):
url_1 = proj_angles[0]
self.assertTrue(url_1.is_valid())
self.assertTrue(url_1.is_absolute())
self.assertEquals(url_1.scheme(), 'silx')
self.assertEqual(url_1.scheme(), 'silx')
# check conversion to dict
_dict = self.scan.to_dict()
scan2 = HDF5TomoScan.from_dict(_dict)
......@@ -180,10 +180,10 @@ class TestHDF5Scan(HDF5TestBaseClass):
self.assertEqual(self.scan.tomo_n, 1500)
radios_urls_evolution = self.scan.get_proj_angle_url()
self.assertEquals(len(radios_urls_evolution), 1503)
self.assertEquals(radios_urls_evolution[0].file_path(), self.scan.master_file)
self.assertEquals(radios_urls_evolution[0].data_slice(), 22)
self.assertEquals(radios_urls_evolution[0].data_path(), 'entry0000/instrument/detector/data')
self.assertEqual(len(radios_urls_evolution), 1503)
self.assertEqual(radios_urls_evolution[0].file_path(), self.scan.master_file)
self.assertEqual(radios_urls_evolution[0].data_slice(), 22)
self.assertEqual(radios_urls_evolution[0].data_path(), 'entry0000/instrument/detector/data')
def testDarkRefUtils(self):
self.assertEqual(self.scan.tomo_n, 1500)
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["W. de Nolf"]
__license__ = "MIT"
__date__ = "25/08/2020"
import h5py
from tomoscan.utils import SharedLockPool
import os
from contextlib import contextmanager
import traceback
import errno
HASSWMR = h5py.version.hdf5_version_tuple >= h5py.get_config().swmr_min_hdf5_version
class HDF5File(h5py.File):
"""File to secure reading and writing within h5py
code originally from bliss.nexus_writer_service.io.nexus
"""
_LOCKPOOL = SharedLockPool()
def __init__(
self, filename, mode, enable_file_locking=None, swmr=None, **kwargs
):
"""
:param str filename:
:param str mode:
:param bool enable_file_locking: by default it is disabled for `mode=='r'`
and enabled in all other modes
:param bool swmr: when not specified: try both modes when `mode=='r'`
:param **kwargs: see `h5py.File.__init__`
"""
if mode not in ('r', 'w', 'w-', 'x', 'a'):
raise ValueError('invalid mode {}'.format(mode))
with self._protect_init(filename):
# https://support.hdfgroup.org/HDF5/docNewFeatures/SWMR/Design-HDF5-FileLocking.pdf
if not HASSWMR and swmr:
swmr = False
libver = kwargs.get("libver")
if swmr:
kwargs["libver"] = "latest"
if enable_file_locking is None:
enable_file_locking = mode != "r"
if enable_file_locking:
os.environ["HDF5_USE_FILE_LOCKING"] = "TRUE"
else:
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
kwargs["track_order"] = True
try:
super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
if mode != "r" and swmr:
# Try setting writing in SWMR mode
try:
self.swmr_mode = True
except Exception:
pass
except OSError as e:
if (
swmr is not None
or mode != "r"
or not HASSWMR
or not isErrno(e, errno.EAGAIN)
):
raise
# Try reading with opposite SWMR mode
swmr = not swmr
if swmr:
kwargs["libver"] = "latest"
else:
kwargs["libver"] = libver
super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
@contextmanager
def _protect_init(self, filename):
"""Makes sure no other file is opened/created
or protected sections associated to the filename
are executed.
"""
lockname = os.path.abspath(filename)
with self._LOCKPOOL.acquire(None):
with self._LOCKPOOL.acquire(lockname):
yield
@contextmanager
def protect(self):
"""Protected section associated to this file.
"""
lockname = os.path.abspath(self.filename)
with self._LOCKPOOL.acquire(lockname):
yield
def isErrno(e, errno):
"""
:param OSError e:
:returns bool:
"""
# Because e.__cause__ is None for chained exceptions
return "errno = {}".format(errno) in "".join(traceback.format_exc())
......@@ -30,6 +30,8 @@ __date__ = "02/10/2017"
import functools
from contextlib import contextmanager, ExitStack
import threading
def _docstring(dest, origin):
......@@ -65,3 +67,55 @@ def docstring(origin):
If the origin class has not method n case the
"""
return functools.partial(_docstring, origin=origin)
class SharedLockPool:
"""
Allows to acquire locks identified by name (hashable type) recursively.
"""
def __init__(self):
self.__locks = {}
self.__locks_mutex = threading.Semaphore(value=1)
def __len__(self):
return len(self.__locks)
@property
def names(self):
return list(self.__locks.keys())
@contextmanager
def _modify_locks(self):
self.__locks_mutex.acquire()
try:
yield self.__locks
finally:
self.__locks_mutex.release()
@contextmanager
def acquire(self, name):
with self._modify_locks() as locks:
lock = locks.get(name, None)
if lock is None:
locks[name] = lock = threading.RLock()
lock.acquire()
try:
yield
finally:
lock.release()
with self._modify_locks() as locks:
locks.pop(name)
@contextmanager
def acquire_context_creation(self, name, contextmngr, *args, **kwargs):
"""
Acquire lock only during context creation.
This can be used for example to protect the opening of a file
but not hold the lock while the file is open.
"""
with ExitStack() as stack:
with self.acquire(name):
ret = stack.enter_context(contextmngr(*args, **kwargs))
yield ret
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment