Merge branch 'fix-h5py3' into 'master'

Fix h5py v3 related issues

Closes #83

See merge request !130

CHANGELOG.rst

@@ -6,8 +6,8 @@ Untagged
 
 * Compatibility:
 
-  - Avoid `h5py` deprecation warnings (MR: !123, !128)
-  - Avoid `silx` 0.13 deprecation warnings (MR: !126, !129)
+  - Fixed `h5py` v3 compatibilty (MR: !130) and deprecation warnings (MR: !123, !128)
+  - Fixed `silx` v0.14 issue (MR: !130) and v0.13 deprecation warnings (MR: !126, !129)
 
 * Miscellaneous:
 

xsocs/gui/widgets/Wizard.py

@@ -111,7 +111,7 @@ class LoadXsocsDataPage(_BaseWizardPage):
             Qt.QMessageBox.critical(self,
                                     'Failed to open project file.',
                                     str(ex))
-            return
+            return False
 
         try:
             projectH5.xsocsFile = xsocsFile
@@ -119,7 +119,7 @@ class LoadXsocsDataPage(_BaseWizardPage):
             Qt.QMessageBox.critical(self,
                                     'Failed to set data file.',
                                     str(ex))
-            return
+            return False
         self.setCommitPage(True)
         return True
 

xsocs/gui/widgets/XsocsPlot2D.py

@@ -36,18 +36,17 @@ from collections import namedtuple, OrderedDict
 
 import numpy as np
 from matplotlib import cm
-from matplotlib.colors import Normalize
+from matplotlib.colors import Normalize, ListedColormap
 
 
 from silx.gui import qt as Qt
+from silx.gui import colors
 from silx.io.utils import savetxt
 from silx.gui.icons import getQIcon
 from silx.gui.plot import PlotWindow
 from silx.math.histogram import Histogramnd
 from silx.gui.widgets.RangeSlider import RangeSlider
 
-from silx.gui.plot.matplotlib import Colormap
-
 from ..widgets.Containers import GroupBox
 from ..widgets.PointWidget import PointWidget
 from ..widgets.Input import StyledLineEdit
@@ -61,6 +60,16 @@ XsocsPlot2DColormap = namedtuple('XsocsPlot2DColormap',
                                  ['colormap', 'minVal', 'maxVal', 'nColors'])
 
 
+def _getViridis():
+    """Get viridis colormap"""
+    try:
+        return cm.viridis
+    except AttributeError:  # Fallback for matplotlib < 1.5.0 (i.e., debian 8)
+        return ListedColormap(
+            colors.Colormap('viridis').getNColors(256).astype(np.float32) / 255.,
+            name='viridis')
+
+
 def _arrayToPixmap(vector, cmap, nColors=256):
     """
     Creates a pixmap from an array, using the provided colormap.
@@ -134,7 +143,7 @@ class XsocsPlot2DColorDialog(Qt.QDialog):
     """
     colormaps = OrderedDict([('jet', cm.jet),
                              ('afmhot', cm.afmhot),
-                             ('viridis', Colormap.getColormap('viridis')),
+                             ('viridis', _getViridis()),
                              ('gray', cm.gray)])
 
     def __init__(self, plot, curve, **kwargs):

xsocs/io/XsocsH5.py

@@ -39,6 +39,11 @@ import numpy as _np
 from .XsocsH5Base import XsocsH5Base
 from ._utils import str_to_h5_utf8, find_NX_class
 
+try:  # silx >= 0.14
+    from silx.io.utils import h5py_read_dataset
+except ImportError:  # Fall back for silx < 0.14
+    from ..util.silx_io_utils import h5py_read_dataset
+
 
 class InvalidEntryError(Exception):
     pass
@@ -80,7 +85,7 @@ class XsocsH5(XsocsH5Base):
     def title(self, entry):
         with self._get_file() as h5_file:
             path = entry + '/title'
-            return h5_file[path][()]
+            return h5py_read_dataset(h5_file[path], decode_ascii=True)
 
     @_process_entry
     def entry_filename(self, entry):

xsocs/io/XsocsH5Base.py

@@ -39,6 +39,11 @@ import numpy as _np
 from ..util import text_type
 from ._utils import str_to_h5_utf8
 
+try:  # silx >= 0.14
+    from silx.io.utils import h5py_read_dataset
+except ImportError:  # Fall back for silx < 0.14
+    from ..util.silx_io_utils import h5py_read_dataset
+
 
 # We have to work around a limitation of the h5py.Group.copy method
 # that fails when a group already exists in the destination file.
@@ -179,7 +184,7 @@ class XsocsH5Base(object):
                     return h5_file[path].shape
                 if dtype:
                     return h5_file[path].dtype
-                return h5_file[path][()]
+                return h5py_read_dataset(h5_file[path], decode_ascii=True)
             except KeyError:
                 return None
 

xsocs/process/merge/KmapMerger.py

@@ -47,6 +47,11 @@ import fabio
 from ...io import XsocsH5
 from ... import config
 
+try:  # silx >= 0.14
+    from silx.io.utils import h5py_read_dataset
+except ImportError:  # Fall back for silx < 0.14
+    from ...util.silx_io_utils import h5py_read_dataset
+
 
 _logger = logging.getLogger(__name__)
 
@@ -397,7 +402,7 @@ class KmapMerger(object):
             except KeyError:
                 raise ValueError('Scan ID {0} not found.'.format(scan_id))
 
-            command = scan['title'][()]
+            command = h5py_read_dataset(scan['title'], decode_ascii=True)
 
         try:
             return parse_scan_command(command)
@@ -426,7 +431,8 @@ class KmapMerger(object):
             calibration = {}
 
             # Parse spec header to find calibration
-            header = scan['instrument/specfile/scan_header'][()]
+            header = h5py_read_dataset(
+                scan['instrument/specfile/scan_header'], decode_ascii=True)
             for line in header:
                 if line.startswith('#UDETCALIB ') or line.startswith('#UMONO '):
                     params = line.split(' ')[1].strip().split(',')
@@ -746,7 +752,8 @@ def _add_edf_data(scan_id,
 
             entry_h5f.copy_group(spec_h5_fn, scan_id, entry)
             with h5py.File(spec_h5_fn, mode='r') as spec_h5:
-                command = spec_h5[scan_id]['title'][()]
+                command = h5py_read_dataset(
+                    spec_h5[scan_id]['title'], decode_ascii=True)
                 command_params = parse_scan_command(command)
 
             entry_h5f.set_scan_params(entry, **command_params)

xsocs/process/merge/KmapSpecParser.py

@@ -39,6 +39,10 @@ from threading import Thread
 
 import h5py
 from silx.io import convert
+try:  # silx >= 0.14
+    from silx.io.utils import h5py_read_dataset
+except ImportError:  # Fall back for silx < 0.14
+    from ...util.silx_io_utils import h5py_read_dataset
 
 
 # regular expression matching the imageFile comment line
@@ -246,7 +250,8 @@ def _spec_get_img_filenames(spec_h5_filename):
         regx = re.compile(_IMAGEFILE_LINE_PATTERN)
 
         for k_scan, v_scan in h5_f.items():
-            header = v_scan['instrument/specfile/scan_header'][()]
+            header = h5py_read_dataset(
+                v_scan['instrument/specfile/scan_header'], decode_ascii=True)
             imgfile_match = [m for line in header
                              if line.startswith('#C imageFile')
                              for m in [regx.match(line.strip())] if m]

xsocs/util/silx_io_utils.py

+# coding: utf-8
+# /*##########################################################################
+# Copyright (C) 2016-2020 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ############################################################################*/
+""" I/O utility functions"""
+
+__authors__ = ["P. Knobel", "V. Valls"]
+__license__ = "MIT"
+__date__ = "25/09/2020"
+
+import logging
+
+import numpy
+
+import h5py
+import h5py.h5t
+
+
+logger = logging.getLogger(__name__)
+
+
+def h5py_decode_value(value, encoding="utf-8", errors="surrogateescape"):
+    """Keep bytes when value cannot be decoded
+
+    :param value: bytes or array of bytes
+    :param encoding str:
+    :param errors str:
+    """
+    try:
+        if numpy.isscalar(value):
+            return value.decode(encoding, errors=errors)
+        str_item = [b.decode(encoding, errors=errors) for b in value.flat]
+        return numpy.array(str_item, dtype=object).reshape(value.shape)
+    except UnicodeDecodeError:
+        return value
+
+
+def h5py_encode_value(value, encoding="utf-8", errors="surrogateescape"):
+    """Keep string when value cannot be encoding
+
+    :param value: string or array of strings
+    :param encoding str:
+    :param errors str:
+    """
+    try:
+        if numpy.isscalar(value):
+            return value.encode(encoding, errors=errors)
+        bytes_item = [s.encode(encoding, errors=errors) for s in value.flat]
+        return numpy.array(bytes_item, dtype=object).reshape(value.shape)
+    except UnicodeEncodeError:
+        return value
+
+
+class H5pyDatasetReadWrapper:
+    """Wrapper to handle H5T_STRING decoding on-the-fly when reading
+    a dataset. Uniform behaviour for h5py 2.x and h5py 3.x
+
+    h5py abuses H5T_STRING with ASCII character set
+    to store `bytes`: dset[()] = b"..."
+    Therefore an H5T_STRING with ASCII encoding is not decoded by default.
+    """
+
+    H5PY_AUTODECODE_NONASCII = int(h5py.version.version.split(".")[0]) < 3
+
+    def __init__(self, dset, decode_ascii=False):
+        """
+        :param h5py.Dataset dset:
+        :param bool decode_ascii:
+        """
+        try:
+            string_info = h5py.h5t.check_string_dtype(dset.dtype)
+        except AttributeError:
+            # h5py < 2.10
+            try:
+                idx = dset.id.get_type().get_cset()
+            except AttributeError:
+                # Not an H5T_STRING
+                encoding = None
+            else:
+                encoding = ["ascii", "utf-8"][idx]
+        else:
+            # h5py >= 2.10
+            try:
+                encoding = string_info.encoding
+            except AttributeError:
+                # Not an H5T_STRING
+                encoding = None
+        if encoding == "ascii" and not decode_ascii:
+            encoding = None
+        if encoding != "ascii" and self.H5PY_AUTODECODE_NONASCII:
+            # Decoding is already done by the h5py library
+            encoding = None
+        if encoding == "ascii":
+            # ASCII can be decoded as UTF-8
+            encoding = "utf-8"
+        self._encoding = encoding
+        self._dset = dset
+
+    def __getitem__(self, args):
+        value = self._dset[args]
+        if self._encoding:
+            return h5py_decode_value(value, encoding=self._encoding)
+        else:
+            return value
+
+
+def h5py_read_dataset(dset, index=tuple(), decode_ascii=False):
+    """Read data from dataset object. UTF-8 strings will be
+    decoded while ASCII strings will only be decoded when
+    `decode_ascii=True`.
+
+    :param h5py.Dataset dset:
+    :param index: slicing (all by default)
+    :param bool decode_ascii:
+    """
+    return H5pyDatasetReadWrapper(dset, decode_ascii=decode_ascii)[index]