io.py 5.94 KB
Newer Older
payno's avatar
payno committed
1
2
# coding: utf-8
# /*##########################################################################
3
#
payno's avatar
payno committed
4
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
5
#
payno's avatar
payno committed
6
7
8
9
10
11
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
12
#
payno's avatar
payno committed
13
14
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
15
#
payno's avatar
payno committed
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/

__authors__ = ["W. de Nolf"]
__license__ = "MIT"
__date__ = "25/08/2020"

Pierre Paleo's avatar
Pierre Paleo committed
30
import logging
31
32
33
34
35
36
37
38
import h5py
from tomoscan.utils import SharedLockPool
import os
from contextlib import contextmanager
import traceback
import errno

HASSWMR = h5py.version.hdf5_version_tuple >= h5py.get_config().swmr_min_hdf5_version
Pierre Paleo's avatar
Pierre Paleo committed
39
_logger = logging.getLogger(__name__)
40
41
42
43
44
45

class HDF5File(h5py.File):
    """File to secure reading and writing within h5py

    code originally from bliss.nexus_writer_service.io.nexus
    """
payno's avatar
payno committed
46

47
48
    _LOCKPOOL = SharedLockPool()

payno's avatar
payno committed
49
    def __init__(self, filename, mode, enable_file_locking=None, swmr=None, **kwargs):
50
51
52
53
54
55
56
57
        """
        :param str filename:
        :param str mode:
        :param bool enable_file_locking: by default it is disabled for `mode=='r'`
                                         and enabled in all other modes
        :param bool swmr: when not specified: try both modes when `mode=='r'`
        :param **kwargs: see `h5py.File.__init__`
        """
payno's avatar
payno committed
58
59
        if mode not in ("r", "w", "w-", "x", "a"):
            raise ValueError("invalid mode {}".format(mode))
60
61
62
63
64
65
66
67
68
69

        with self._protect_init(filename):
            # https://support.hdfgroup.org/HDF5/docNewFeatures/SWMR/Design-HDF5-FileLocking.pdf
            if not HASSWMR and swmr:
                swmr = False
            libver = kwargs.get("libver")
            if swmr:
                kwargs["libver"] = "latest"
            if enable_file_locking is None:
                enable_file_locking = mode != "r"
70
            old_file_locking = os.environ.get("HDF5_USE_FILE_LOCKING", None)
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
            if enable_file_locking:
                os.environ["HDF5_USE_FILE_LOCKING"] = "TRUE"
            else:
                os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
            kwargs["track_order"] = True
            try:
                super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
                if mode != "r" and swmr:
                    # Try setting writing in SWMR mode
                    try:
                        self.swmr_mode = True
                    except Exception:
                        pass
            except OSError as e:
                if (
                    swmr is not None
                    or mode != "r"
                    or not HASSWMR
                    or not isErrno(e, errno.EAGAIN)
                ):
                    raise
                # Try reading with opposite SWMR mode
                swmr = not swmr
                if swmr:
                    kwargs["libver"] = "latest"
                else:
                    kwargs["libver"] = libver
                super().__init__(filename, mode=mode, swmr=swmr, **kwargs)
99
100
101
102
            if old_file_locking is None:
                del os.environ["HDF5_USE_FILE_LOCKING"]
            else:
                os.environ["HDF5_USE_FILE_LOCKING"] = old_file_locking
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    @contextmanager
    def _protect_init(self, filename):
        """Makes sure no other file is opened/created
        or protected sections associated to the filename
        are executed.
        """
        lockname = os.path.abspath(filename)
        with self._LOCKPOOL.acquire(None):
            with self._LOCKPOOL.acquire(lockname):
                yield

    @contextmanager
    def protect(self):
payno's avatar
payno committed
117
        """Protected section associated to this file."""
118
119
120
121
122
123
124
125
126
127
128
129
        lockname = os.path.abspath(self.filename)
        with self._LOCKPOOL.acquire(lockname):
            yield


def isErrno(e, errno):
    """
    :param OSError e:
    :returns bool:
    """
    # Because e.__cause__ is None for chained exceptions
    return "errno = {}".format(errno) in "".join(traceback.format_exc())
Pierre Paleo's avatar
Pierre Paleo committed
130

Pierre Paleo's avatar
Pierre Paleo committed
131

Pierre Paleo's avatar
Pierre Paleo committed
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def check_virtual_sources_exist(fname, data_path):
    """
    Check that a virtual dataset points to actual data.

    Parameters
    ----------
    fname: str
        HDF5 file path
    data_path: str
        Path within the HDF5 file

    Returns
    --------
    res: bool
        Whether the virtual dataset points to actual data.
    """
    with HDF5File(fname, "r") as f:
        if data_path not in f:
Pierre Paleo's avatar
Pierre Paleo committed
150
            _logger.error("No dataset %s in file %s" % (data_path, fname))
Pierre Paleo's avatar
Pierre Paleo committed
151
152
153
154
155
            return False
        dptr = f[data_path]
        if not dptr.is_virtual:
            return True
        for vsource in dptr.virtual_sources():
Pierre Paleo's avatar
Pierre Paleo committed
156
157
158
            vsource_fname = os.path.join(
                os.path.dirname(dptr.file.filename), vsource.file_name
            )
Pierre Paleo's avatar
Pierre Paleo committed
159
            if not os.path.isfile(vsource_fname):
Pierre Paleo's avatar
Pierre Paleo committed
160
                _logger.error("No such file: %s" % vsource_fname)
Pierre Paleo's avatar
Pierre Paleo committed
161
162
                return False
            elif not check_virtual_sources_exist(vsource_fname, vsource.dset_name):
Pierre Paleo's avatar
Pierre Paleo committed
163
                _logger.error("Error with virtual source %s" % vsource_fname)
Pierre Paleo's avatar
Pierre Paleo committed
164
165
                return False
    return True