Merge branch 'master' of gitlab.esrf.fr:tomotools/tomoscan

.gitlab-ci.yml

@@ -19,7 +19,6 @@ check_style:
   before_script:
     - pip install black
   script:
-    - pip install -r requirements.txt
     # run black
     - LC_ALL=C.UTF-8 black --check --safe .
 
@@ -34,12 +33,14 @@ doc:
     - arch
     - which python
     - python --version
+    - source ./ci/install_scripts.sh
     - python -m pip install pip --upgrade
     - python -m pip install setuptools --upgrade
     - python -m pip install sphinx
     - python -m pip install nbsphinx
     - python -m pip install nbconvert
     - python -m pip install pandoc
+    - install_silx
     - python -m pip install -r requirements.txt
     - rm -rf artifacts
     - mkdir artifacts
@@ -66,19 +67,16 @@ doc:
     - export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
     - export ORANGE_WEB_LOG='False'
     - python --version
+    - source ./ci/install_scripts.sh
     - python -m pip install pip --upgrade
     - python -m pip install setuptools --upgrade
+    - install_silx
     - python -m pip install -r requirements.txt
     - python -m pip install .
   script:
     - python -m unittest tomoscan.test.suite -v
 
 
-test:python3.5-stretch-pyqt5:
-  image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
-  <<: *test_linux_template
-
-
 test:python3.7_stretch_pyqt5:
   image: docker-registry.esrf.fr/dau/tomwer:python3.7_stretch_pyqt5
   <<: *test_linux_template
@@ -95,6 +93,7 @@ test:test-tomoscan-tutorials:
     - export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
     - export ORANGE_WEB_LOG='False'
     - python --version
+    - source ./ci/install_scripts.sh
     - python -m pip install pip --upgrade
     - python -m pip install setuptools --upgrade
     - python -m pip install numpy --upgrade
@@ -103,6 +102,7 @@ test:test-tomoscan-tutorials:
     - python -m pip install nbconvert
     - python -m pip install pandoc
     - python -m pip install ipykernel
+    - install_silx
     - python -m pip install -r requirements.txt
     - python -m pip install fabio --upgrade --pre
     - python -m pip install silx --upgrade --pre

CHANGELOG.rst

 Change Log
 ==========
 
-0.3.2: XXX
-----------
+0.4.0: 2020/11/09
+-----------------
+
+* esrf
+    * HDF5TomoScan & EDFTomoScan add ignore_projections in the constructor
+    * add properties to scanBase.TomoScanBase class:
+        * alignment_projections
+        * estimated_cor_frm_motor
+
+    * add flat_field_correction function
+    * move h5py dependency to 3.x
+
+* Miscellaneous: add 'black' format
+
+0.3.2: 2020/08/26
+-----------------
 
 * HDF5TomoScan
     * manage hdf5 files using the HDFFile Class in order to avoid any file corruption.

ci/install_scripts.sh

+#!/bin/bash
+
+function install_silx(){
+
+    if [ "$2" = 'latest' ]; then
+        python -m install silx
+    else
+        python -m pip install git+https://github.com/silx-kit/silx.git
+    fi
+}
+
+
+function silx_version(){
+    python -c 'import silx; print(silx.version)'
+}

requirements.txt

-silx>=0.9
+# silx>=0.14
 lxml
-numpy
\ No newline at end of file
+numpy
+h5py >= 3.0
\ No newline at end of file

setup.py

@@ -321,8 +321,9 @@ def get_project_configuration(dry_run):
     install_requires = [
         # for the script launcher and pkg_resources
         "setuptools",
-        "silx>=0.9",
+        "silx >= 0.14a",
         "lxml",
+        "h5py >= 3.0",
     ]
 
     setup_requires = [

tomoscan/esrf/hdf5scan.py

@@ -43,6 +43,7 @@ from tomoscan.io import HDF5File
 from silx.io.utils import get_data
 from ..unitsystem import metricsystem
 from .utils import get_compacted_dataslices
+from silx.io.utils import h5py_read_dataset
 import typing
 import logging
 
@@ -267,6 +268,13 @@ class HDF5TomoScan(TomoScanBase):
         if "definition" in node.attrs and node.attrs["definition"].lower() == "nxtomo":
             _logger.info(node.name + " is recognized as an NXtomo class.")
             return True
+        elif (
+            "instrument" in node
+            and "NX_class" in node["instrument"].attrs
+            and node["instrument"].attrs["NX_class"] == "NXinstrument"
+        ):
+            instrument_node = node["instrument"]
+            return "detector" in node["instrument"]
         else:
             return False
 
@@ -455,7 +463,9 @@ class HDF5TomoScan(TomoScanBase):
         if self._rotation_angles is None:
             self._check_hdf5scan_validity()
             with HDF5File(self.master_file, "r", swmr=True) as h5_file:
-                _rotation_angles = h5_file[self._entry][self._ROTATION_ANGLE_PATH][()]
+                _rotation_angles = h5py_read_dataset(
+                    h5_file[self._entry][self._ROTATION_ANGLE_PATH]
+                )
                 # cast in float
                 self._rotation_angles = tuple(
                     [float(angle) for angle in _rotation_angles]
@@ -467,7 +477,9 @@ class HDF5TomoScan(TomoScanBase):
         if self._entry and self._image_keys is None:
             self._check_hdf5scan_validity()
             with HDF5File(self.master_file, "r", swmr=True) as h5_file:
-                self._image_keys = h5_file[self._entry][self._IMG_KEY_PATH][()]
+                self._image_keys = h5py_read_dataset(
+                    h5_file[self._entry][self._IMG_KEY_PATH]
+                )
         return self._image_keys
 
     @property
@@ -476,9 +488,9 @@ class HDF5TomoScan(TomoScanBase):
             self._check_hdf5scan_validity()
             with HDF5File(self.master_file, "r", swmr=True) as h5_file:
                 if self._IMG_KEY_CONTROL_PATH in h5_file[self._entry]:
-                    self._image_keys_control = h5_file[self._entry][
-                        self._IMG_KEY_CONTROL_PATH
-                    ][()]
+                    self._image_keys_control = h5py_read_dataset(
+                        h5_file[self._entry][self._IMG_KEY_CONTROL_PATH]
+                    )
                 else:
                     self._image_keys_control = None
         return self._image_keys_control
@@ -575,7 +587,7 @@ class HDF5TomoScan(TomoScanBase):
     def _get_fov(self):
         with HDF5File(self.master_file, "r", swmr=True, libver="latest") as h5_file:
             if self._FOV_PATH in h5_file[self._entry]:
-                fov = h5_file[self._entry][self._FOV_PATH][()]
+                fov = h5py_read_dataset(h5_file[self._entry][self._FOV_PATH])
                 return _FOV.from_value(fov)
             else:
                 return None
@@ -583,7 +595,9 @@ class HDF5TomoScan(TomoScanBase):
     def _get_estimated_cor_frm_motor(self):
         with HDF5File(self.master_file, "r", swmr=True, libver="latest") as h5_file:
             if self._ESTIMATED_COR_FRM_MOTOR_PATH in h5_file[self._entry]:
-                value = h5_file[self._entry][self._ESTIMATED_COR_FRM_MOTOR_PATH][()]
+                value = h5py_read_dataset(
+                    h5_file[self._entry][self._ESTIMATED_COR_FRM_MOTOR_PATH]
+                )
                 return float(value)
             else:
                 return None
@@ -793,7 +807,7 @@ class HDF5TomoScan(TomoScanBase):
         Unit can be defined in on of the group attributes. It it is the case
         will pick this unit, otherwise will use the default unit
         """
-        value = node[()]
+        value = h5py_read_dataset(node)
         if "unit" in node.attrs:
             unit = node.attrs["unit"]
         elif "units" in node.attrs:

tomoscan/esrf/mock.py

@@ -36,6 +36,7 @@ from xml.etree import cElementTree
 import fabio
 import fabio.edfimage
 from .hdf5scan import ImageKey, HDF5TomoScan
+from silx.io.utils import h5py_read_dataset
 
 
 class _ScanMock:
@@ -128,7 +129,7 @@ class _ScanMock:
             radius = min(background.shape)
 
             def _compute_radius_to_center(data):
-                assert data.ndim is 2
+                assert data.ndim == 2
                 xcenter = (data.shape[2]) // 2
                 ycenter = (data.shape[1]) // 2
                 y, x = numpy.ogrid[: data.shape[0], : data.shape[1]]
@@ -293,7 +294,7 @@ class MockHDF5(_ScanMock):
             # add data
             if "data" in detector_grp:
                 # read and remove data
-                current_dataset = detector_grp["data"][()]
+                current_dataset = h5py_read_dataset(detector_grp["data"])
                 new_dataset = numpy.append(current_dataset, data_)
                 del detector_grp["data"]
                 shape = list(current_dataset.shape)
@@ -304,7 +305,7 @@ class MockHDF5(_ScanMock):
 
             # add rotation angle
             if "rotation_angle" in sample_grp:
-                new_rot_angle = sample_grp["rotation_angle"][()]
+                new_rot_angle = h5py_read_dataset(sample_grp["rotation_angle"])
                 new_rot_angle = numpy.append(new_rot_angle, rotation_angle)
                 del sample_grp["rotation_angle"]
             else:
@@ -314,7 +315,7 @@ class MockHDF5(_ScanMock):
 
             # add image_key
             if "image_key" in detector_grp:
-                new_image_key = detector_grp["image_key"][()]
+                new_image_key = h5py_read_dataset(detector_grp["image_key"])
                 new_image_key = numpy.append(new_image_key, image_key)
                 del detector_grp["image_key"]
             else:
@@ -324,7 +325,9 @@ class MockHDF5(_ScanMock):
 
             # add image_key_control
             if "image_key_control" in detector_grp:
-                new_image_key_control = detector_grp["image_key_control"][()]
+                new_image_key_control = h5py_read_dataset(
+                    detector_grp["image_key_control"]
+                )
                 new_image_key_control = numpy.append(
                     new_image_key_control, image_key_control
                 )
@@ -336,7 +339,7 @@ class MockHDF5(_ScanMock):
 
             # add count_time
             if "count_time" in detector_grp:
-                new_count_time = detector_grp["count_time"][()]
+                new_count_time = h5py_read_dataset(detector_grp["count_time"])
                 new_count_time = numpy.append(new_count_time, self._PROJ_COUNT)
                 del detector_grp["count_time"]
             else:
@@ -530,7 +533,7 @@ class MockEDF(_ScanMock):
         S1=0.0,
         S2=1.0,
     ):
-        assert len(shape) is 3
+        assert len(shape) == 3
         f = open(filePath, "w")
         f.writelines(
             "\n".join(

tomoscan/scanbase.py

@@ -414,7 +414,7 @@ class TomoScanBase:
                         "incoherent data information to retrieve"
                         "scan extra images angle"
                     )
-                elif len(extraImgs) is 3:
+                elif len(extraImgs) == 3:
                     res["180(1)"] = ordered_url[extraImgs[0]]
                     res["90(1)"] = ordered_url[extraImgs[1]]
                     res["0(1)"] = ordered_url[extraImgs[2]]

tomoscan/unitsystem/metricsystem.py

@@ -98,7 +98,7 @@ class MetricSystem(_Enum):
         if isinstance(value, str):
             return cls.from_str(value=value)
         else:
-            _Enum.from_value(value=value)
+            return _Enum.from_value(value=value)
 
     @classmethod
     def from_str(cls, value: str):
@@ -124,6 +124,28 @@ class MetricSystem(_Enum):
         else:
             raise ValueError("Cannot convert: %s" % value)
 
+    def __str__(self):
+        if self == MetricSystem.METER:
+            return "m"
+        elif self == MetricSystem.CENTIMETER:
+            return "cm"
+        elif self == MetricSystem.MILLIMETER:
+            return "mm"
+        elif self == MetricSystem.MICROMETER:
+            return "um"
+        elif self == MetricSystem.NANOMETER:
+            return "nm"
+        elif self == MetricSystem.KILOELECTRONVOLT:
+            return "keV"
+        elif self == MetricSystem.ELECTRONVOLT:
+            return "eV"
+        elif self == MetricSystem.JOULE:
+            return "J"
+        elif self == MetricSystem.KILOJOULE:
+            return "kJ"
+        else:
+            raise ValueError("Cannot convert: %s" % value)
+
 
 m = MetricSystem.METER
 meter = MetricSystem.METER