Commit e81b7a1e authored by Thomas Vincent's avatar Thomas Vincent
Browse files

Merge branch 'fix-h5py3' into 'master'

Fix h5py v3 related issues

Closes #83

See merge request !130
parents 986fe9c9 d37746be
Pipeline #37104 passed with stages
in 9 minutes
......@@ -6,8 +6,8 @@ Untagged
* Compatibility:
- Avoid `h5py` deprecation warnings (MR: !123, !128)
- Avoid `silx` 0.13 deprecation warnings (MR: !126, !129)
- Fixed `h5py` v3 compatibilty (MR: !130) and deprecation warnings (MR: !123, !128)
- Fixed `silx` v0.14 issue (MR: !130) and v0.13 deprecation warnings (MR: !126, !129)
* Miscellaneous:
......
......@@ -111,7 +111,7 @@ class LoadXsocsDataPage(_BaseWizardPage):
Qt.QMessageBox.critical(self,
'Failed to open project file.',
str(ex))
return
return False
try:
projectH5.xsocsFile = xsocsFile
......@@ -119,7 +119,7 @@ class LoadXsocsDataPage(_BaseWizardPage):
Qt.QMessageBox.critical(self,
'Failed to set data file.',
str(ex))
return
return False
self.setCommitPage(True)
return True
......
......@@ -36,18 +36,17 @@ from collections import namedtuple, OrderedDict
import numpy as np
from matplotlib import cm
from matplotlib.colors import Normalize
from matplotlib.colors import Normalize, ListedColormap
from silx.gui import qt as Qt
from silx.gui import colors
from silx.io.utils import savetxt
from silx.gui.icons import getQIcon
from silx.gui.plot import PlotWindow
from silx.math.histogram import Histogramnd
from silx.gui.widgets.RangeSlider import RangeSlider
from silx.gui.plot.matplotlib import Colormap
from ..widgets.Containers import GroupBox
from ..widgets.PointWidget import PointWidget
from ..widgets.Input import StyledLineEdit
......@@ -61,6 +60,16 @@ XsocsPlot2DColormap = namedtuple('XsocsPlot2DColormap',
['colormap', 'minVal', 'maxVal', 'nColors'])
def _getViridis():
"""Get viridis colormap"""
try:
return cm.viridis
except AttributeError: # Fallback for matplotlib < 1.5.0 (i.e., debian 8)
return ListedColormap(
colors.Colormap('viridis').getNColors(256).astype(np.float32) / 255.,
name='viridis')
def _arrayToPixmap(vector, cmap, nColors=256):
"""
Creates a pixmap from an array, using the provided colormap.
......@@ -134,7 +143,7 @@ class XsocsPlot2DColorDialog(Qt.QDialog):
"""
colormaps = OrderedDict([('jet', cm.jet),
('afmhot', cm.afmhot),
('viridis', Colormap.getColormap('viridis')),
('viridis', _getViridis()),
('gray', cm.gray)])
def __init__(self, plot, curve, **kwargs):
......
......@@ -39,6 +39,11 @@ import numpy as _np
from .XsocsH5Base import XsocsH5Base
from ._utils import str_to_h5_utf8, find_NX_class
try: # silx >= 0.14
from silx.io.utils import h5py_read_dataset
except ImportError: # Fall back for silx < 0.14
from ..util.silx_io_utils import h5py_read_dataset
class InvalidEntryError(Exception):
pass
......@@ -80,7 +85,7 @@ class XsocsH5(XsocsH5Base):
def title(self, entry):
with self._get_file() as h5_file:
path = entry + '/title'
return h5_file[path][()]
return h5py_read_dataset(h5_file[path], decode_ascii=True)
@_process_entry
def entry_filename(self, entry):
......
......@@ -39,6 +39,11 @@ import numpy as _np
from ..util import text_type
from ._utils import str_to_h5_utf8
try: # silx >= 0.14
from silx.io.utils import h5py_read_dataset
except ImportError: # Fall back for silx < 0.14
from ..util.silx_io_utils import h5py_read_dataset
# We have to work around a limitation of the h5py.Group.copy method
# that fails when a group already exists in the destination file.
......@@ -179,7 +184,7 @@ class XsocsH5Base(object):
return h5_file[path].shape
if dtype:
return h5_file[path].dtype
return h5_file[path][()]
return h5py_read_dataset(h5_file[path], decode_ascii=True)
except KeyError:
return None
......
......@@ -47,6 +47,11 @@ import fabio
from ...io import XsocsH5
from ... import config
try: # silx >= 0.14
from silx.io.utils import h5py_read_dataset
except ImportError: # Fall back for silx < 0.14
from ...util.silx_io_utils import h5py_read_dataset
_logger = logging.getLogger(__name__)
......@@ -397,7 +402,7 @@ class KmapMerger(object):
except KeyError:
raise ValueError('Scan ID {0} not found.'.format(scan_id))
command = scan['title'][()]
command = h5py_read_dataset(scan['title'], decode_ascii=True)
try:
return parse_scan_command(command)
......@@ -426,7 +431,8 @@ class KmapMerger(object):
calibration = {}
# Parse spec header to find calibration
header = scan['instrument/specfile/scan_header'][()]
header = h5py_read_dataset(
scan['instrument/specfile/scan_header'], decode_ascii=True)
for line in header:
if line.startswith('#UDETCALIB ') or line.startswith('#UMONO '):
params = line.split(' ')[1].strip().split(',')
......@@ -746,7 +752,8 @@ def _add_edf_data(scan_id,
entry_h5f.copy_group(spec_h5_fn, scan_id, entry)
with h5py.File(spec_h5_fn, mode='r') as spec_h5:
command = spec_h5[scan_id]['title'][()]
command = h5py_read_dataset(
spec_h5[scan_id]['title'], decode_ascii=True)
command_params = parse_scan_command(command)
entry_h5f.set_scan_params(entry, **command_params)
......
......@@ -39,6 +39,10 @@ from threading import Thread
import h5py
from silx.io import convert
try: # silx >= 0.14
from silx.io.utils import h5py_read_dataset
except ImportError: # Fall back for silx < 0.14
from ...util.silx_io_utils import h5py_read_dataset
# regular expression matching the imageFile comment line
......@@ -246,7 +250,8 @@ def _spec_get_img_filenames(spec_h5_filename):
regx = re.compile(_IMAGEFILE_LINE_PATTERN)
for k_scan, v_scan in h5_f.items():
header = v_scan['instrument/specfile/scan_header'][()]
header = h5py_read_dataset(
v_scan['instrument/specfile/scan_header'], decode_ascii=True)
imgfile_match = [m for line in header
if line.startswith('#C imageFile')
for m in [regx.match(line.strip())] if m]
......
# coding: utf-8
# /*##########################################################################
# Copyright (C) 2016-2020 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ############################################################################*/
""" I/O utility functions"""
__authors__ = ["P. Knobel", "V. Valls"]
__license__ = "MIT"
__date__ = "25/09/2020"
import logging
import numpy
import h5py
import h5py.h5t
logger = logging.getLogger(__name__)
def h5py_decode_value(value, encoding="utf-8", errors="surrogateescape"):
"""Keep bytes when value cannot be decoded
:param value: bytes or array of bytes
:param encoding str:
:param errors str:
"""
try:
if numpy.isscalar(value):
return value.decode(encoding, errors=errors)
str_item = [b.decode(encoding, errors=errors) for b in value.flat]
return numpy.array(str_item, dtype=object).reshape(value.shape)
except UnicodeDecodeError:
return value
def h5py_encode_value(value, encoding="utf-8", errors="surrogateescape"):
"""Keep string when value cannot be encoding
:param value: string or array of strings
:param encoding str:
:param errors str:
"""
try:
if numpy.isscalar(value):
return value.encode(encoding, errors=errors)
bytes_item = [s.encode(encoding, errors=errors) for s in value.flat]
return numpy.array(bytes_item, dtype=object).reshape(value.shape)
except UnicodeEncodeError:
return value
class H5pyDatasetReadWrapper:
"""Wrapper to handle H5T_STRING decoding on-the-fly when reading
a dataset. Uniform behaviour for h5py 2.x and h5py 3.x
h5py abuses H5T_STRING with ASCII character set
to store `bytes`: dset[()] = b"..."
Therefore an H5T_STRING with ASCII encoding is not decoded by default.
"""
H5PY_AUTODECODE_NONASCII = int(h5py.version.version.split(".")[0]) < 3
def __init__(self, dset, decode_ascii=False):
"""
:param h5py.Dataset dset:
:param bool decode_ascii:
"""
try:
string_info = h5py.h5t.check_string_dtype(dset.dtype)
except AttributeError:
# h5py < 2.10
try:
idx = dset.id.get_type().get_cset()
except AttributeError:
# Not an H5T_STRING
encoding = None
else:
encoding = ["ascii", "utf-8"][idx]
else:
# h5py >= 2.10
try:
encoding = string_info.encoding
except AttributeError:
# Not an H5T_STRING
encoding = None
if encoding == "ascii" and not decode_ascii:
encoding = None
if encoding != "ascii" and self.H5PY_AUTODECODE_NONASCII:
# Decoding is already done by the h5py library
encoding = None
if encoding == "ascii":
# ASCII can be decoded as UTF-8
encoding = "utf-8"
self._encoding = encoding
self._dset = dset
def __getitem__(self, args):
value = self._dset[args]
if self._encoding:
return h5py_decode_value(value, encoding=self._encoding)
else:
return value
def h5py_read_dataset(dset, index=tuple(), decode_ascii=False):
"""Read data from dataset object. UTF-8 strings will be
decoded while ASCII strings will only be decoded when
`decode_ascii=True`.
:param h5py.Dataset dset:
:param index: slicing (all by default)
:param bool decode_ascii:
"""
return H5pyDatasetReadWrapper(dset, decode_ascii=decode_ascii)[index]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment