normalization: test + bug fixes

.gitlab-ci.yml

@@ -72,6 +72,7 @@ tomwer:
     - python -m pip install .[full_no_cuda]
     - python -m pip install PyQt5==5.11.3
     - python -m pip install pyqtgraph==0.11.0
+    - python -m pip install git+https://gitlab.esrf.fr/tomotools/tomoscan@update_normalization
     - python -m pip install pytest-cov
     - python -m pip install git+https://gitlab.esrf.fr/tomotools/nabu
     - python -m tomwer --help
@@ -93,6 +94,7 @@ test:test-tomwer-tutorials:
     - python -m pip install fabio --upgrade --pre
     - python -m pip install silx --upgrade --pre
     - python -m pip install .[full_no_cuda]
+    - python -m pip install git+https://gitlab.esrf.fr/tomotools/tomoscan@update_normalization
     - mkdir tmp_dir
     - cd tmp_dir
   script:

orangecontrib/tomwer/widgets/reconstruction/NormIOW.py

@@ -156,8 +156,10 @@ class NormIOW(WidgetLongProcessing, SuperviseOW):
         self._layout.setContentsMargins(0, 0, 0, 0)
 
         self._layout.addWidget(self._window)
-
-        self.loadSettings()
+        try:
+            self.loadSettings()
+        except Exception as e:
+            _logger.warning(f"Failed to load settings: {e}")
 
         # connect signal / slot
         self._window.sigConfigurationChanged.connect(self._updateSettings)

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -39,10 +39,16 @@ from tomwer.core.cluster.cluster import (
 )
 from tomwer.core.process.reconstruction.nabu.target import Target
 from tomwer.core.process.reconstruction.nabu.utils import _NabuPhaseMethod
+from tomwer.core.scan.edfscan import EDFTomoScan
+from tomwer.core.scan.hdf5scan import HDF5TomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.slurm import is_slurm_available
+from tomwer.core.process.reconstruction.normalization.params import (
+    _ValueSource as INormSource,
+)
 from tomoscan.normalization import Method as INormMethod
 from silx.utils.enum import Enum as _Enum
+from silx.io.url import DataUrl
 from .slurm import _exec_nabu_on_slurm
 from typing import Iterable, Optional
 from typing import Union
@@ -52,6 +58,8 @@ from . import settings
 from . import utils
 from time import sleep
 import os
+import uuid
+from tomoscan.io import HDF5File
 
 _logger = logging.getLogger(__name__)
 try:
@@ -468,19 +476,67 @@ class _NabuBaseReconstructor:
             config["preproc"] = {}
         if self.scan.intensity_normalization.method is INormMethod.NONE:
             config["preproc"]["sino_normalization"] = ""
-        elif self.scan.intensity_normalization.method is INormMethod.SCALAR:
-            config["preproc"]["sino_normalization"] = "dataset"
-            # TODO: save scalar in a .hdf5 to pass it to Nabu
-            raise NotImplementedError("Scalar not implemented at nabu side for now")
-        elif self.scan.intensity_normalization.method in (
-            INormMethod.CHEBYSHEV,
-            INormMethod.LSQR_SPLINE,
-            INormMethod.MONITOR,
-        ):
+        else:
             config["preproc"][
                 "sino_normalization"
             ] = self.scan.intensity_normalization.method.value
 
+        extra_infos = self.scan.intensity_normalization.get_extra_infos()
+
+        nabu_cfg_folders = os.path.join(
+            config["output"]["location"], settings.NABU_CFG_FILE_FOLDER
+        )
+        os.makedirs(nabu_cfg_folders, exist_ok=True)
+        serving_hatch_file = os.path.join(
+            nabu_cfg_folders, "nabu_tomwer_serving_hatch.h5"
+        )
+
+        source = extra_infos.get("source", INormSource.NONE)
+        source = INormSource.from_value(source)
+
+        if source is INormSource.NONE:
+            pass
+        elif source is INormSource.MANUAL_SCALAR:
+            if "value" not in extra_infos:
+                raise KeyError(
+                    "value should be provided in extra)infos for scalar defined manually"
+                )
+            else:
+                # save the value to a dedicated path in "nabu_tomwer_serving_hatch"
+                if isinstance(self.scan, HDF5TomoScan):
+                    entry_path = self.scan.entry
+                elif isinstance(self.scan, EDFTomoScan):
+                    entry_path = "entry"
+                else:
+                    raise TypeError
+                with HDF5File(serving_hatch_file, mode="a") as h5f:
+                    serving_hatch_data_path = None
+                    # create a unique dataset path to avoid possible conflicts
+                    while (
+                        serving_hatch_data_path is None
+                        or serving_hatch_data_path in h5f
+                    ):
+                        serving_hatch_data_path = "/".join(
+                            [entry_path, str(uuid.uuid1())]
+                        )
+                    h5f[serving_hatch_data_path] = extra_infos["value"]
+
+                serving_hatch_url = DataUrl(
+                    file_path="nabu_tomwer_serving_hatch.h5",  # configuration file and nabu_tomwer_serving_hatch are in the same folder
+                    data_path=serving_hatch_data_path,
+                    scheme="silx",
+                )
+                config["preproc"]["sino_normalization_file"] = serving_hatch_url.path()
+        elif source is INormSource.DATASET:
+            url = extra_infos["dataset_url"]
+            if not isinstance(url, DataUrl):
+                raise TypeError(
+                    f"dataset_url is expected to be an instance of DataUrl. Not {type(url)}"
+                )
+            config["preproc"]["sino_normalization_file"] = url.path()
+        else:
+            raise NotImplementedError(f"source type {source.value} is not handled")
+
         return config
 
     def _get_file_basename_reconstruction(self, pag, db):

tomwer/core/process/reconstruction/nabu/test/test_nabunormalization.py

+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "29/03/2022"
+
+
+import configparser
+from silx.io.url import DataUrl
+from collections import namedtuple
+import pytest
+from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSlices
+from tomwer.core.process.reconstruction.normalization.params import (
+    IntensityNormalizationParams,
+)
+from tomwer.core.process.reconstruction.normalization.normalization import (
+    IntensityNormalizationTask,
+)
+from tomwer.core.utils.scanutils import MockHDF5
+from tomwer.core.process.reconstruction.nabu import settings as nabu_settings
+from tomoscan.io import HDF5File
+import numpy
+import os
+
+_norm_setting = namedtuple("_norm_setting", ["method", "source", "extra_infos"])
+
+_nabu_norm_field = namedtuple(
+    "_nabu_norm_field",
+    ["sino_normalization", "has_sino_normalization_file"],
+)
+
+
+normalization_config_test = (
+    # no normalization
+    (
+        _norm_setting(method="none", source=None, extra_infos=None),
+        _nabu_norm_field(
+            sino_normalization="",
+            has_sino_normalization_file=False,
+        ),
+    ),
+    # chebyshev normalization
+    (
+        _norm_setting(method="chebyshev", source=None, extra_infos=None),
+        _nabu_norm_field(
+            sino_normalization="chebyshev",
+            has_sino_normalization_file=False,
+        ),
+    ),
+    # divide by a scalar
+    (
+        _norm_setting(method="division", source="scalar", extra_infos={"value": 12.0}),
+        _nabu_norm_field(
+            sino_normalization="division",
+            has_sino_normalization_file=True,
+        ),
+    ),
+    # substract a roi
+    (
+        _norm_setting(
+            method="substraction",
+            source="manual ROI",
+            extra_infos={
+                "start_x": 0.0,
+                "end_x": 1.0,
+                "start_y": 0.0,
+                "end_y": 1.0,
+                "calc_fct": "mean",
+                "calc_area": "volume",
+                "calc_method": "scalar",
+            },
+        ),
+        _nabu_norm_field(
+            sino_normalization="substraction",
+            has_sino_normalization_file=True,
+        ),
+    ),
+    # substract from a dataset
+    (
+        _norm_setting(
+            method="substraction",
+            source="from dataset",
+            extra_infos={
+                "dataset_url": DataUrl(
+                    file_path="random_dataset.hdf5",
+                    data_path="data",
+                    scheme="silx",
+                ),
+                "calc_fct": "median",
+                "calc_area": "volume",
+                "calc_method": "scalar",
+            },
+        ),
+        _nabu_norm_field(
+            sino_normalization="substraction",
+            has_sino_normalization_file=True,
+        ),
+    ),
+)
+
+
+@pytest.mark.parametrize(
+    "norm_setting, expected_nabu_conf", [item for item in normalization_config_test]
+)
+def test_normalization(norm_setting, expected_nabu_conf, tmp_path):
+    """
+    Insure normalization is correctly provided to nabu configuration file
+    For this run a normalization process followed by a nabu process.
+    """
+    scan_dir = tmp_path / "scan"
+    scan_dir.mkdir()
+
+    nabu_cfg_folders = os.path.join(scan_dir, nabu_settings.NABU_CFG_FILE_FOLDER)
+    os.makedirs(nabu_cfg_folders, exist_ok=True)
+
+    random_dataset_file = os.path.join(nabu_cfg_folders, "random_dataset.hdf5")
+
+    # create a random dataset if necessary
+    with HDF5File(random_dataset_file, mode="w") as h5f:
+        h5f["data"] = numpy.ones((20, 20))
+
+    mock = MockHDF5(
+        scan_path=scan_dir,
+        n_proj=10,
+        n_ini_proj=10,
+        scan_range=180,
+        dim=20,
+    )
+    scan = mock.scan
+
+    cfg_folder = os.path.join(str(scan_dir), "nabu_cfg_files")
+    cfg_file = os.path.join(cfg_folder, "entry_scan.cfg")
+    assert not os.path.exists(cfg_file)
+    norm_params = IntensityNormalizationParams(
+        method=norm_setting.method,
+        source=norm_setting.source,
+        extra_infos=norm_setting.extra_infos,
+    )
+
+    normalization = IntensityNormalizationTask(
+        inputs={
+            "data": scan,
+            "configuration": norm_params.to_dict(),
+        },
+        varinfo=None,
+    )
+    normalization.run()
+
+    # insure the method is style valid
+    assert scan.intensity_normalization.method.value == norm_setting.method
+    assert hasattr(scan.intensity_normalization, "tomwer_processing_res_code")
+
+    process = NabuSlices(
+        inputs={
+            "data": scan,
+            "dry_run": True,
+        },
+        varinfo=None,
+    )
+    process.run()
+    assert os.path.exists(cfg_file)
+
+    configuration = configparser.ConfigParser(allow_no_value=True)
+    configuration.read(cfg_file)
+
+    preproc_section = configuration["preproc"]
+    sino_normalization = preproc_section.get("sino_normalization", "")
+    sino_normalization_file = preproc_section.get("sino_normalization_file", "")
+    assert sino_normalization == expected_nabu_conf.sino_normalization
+    if expected_nabu_conf.has_sino_normalization_file:
+        url = DataUrl(path=sino_normalization_file)
+        assert url.is_valid()
+    else:
+        assert sino_normalization_file == ""
+    assert hasattr(scan.intensity_normalization, "tomwer_processing_res_code")

tomwer/core/process/reconstruction/normalization/__init__.py

 from .normalization import (  # noqa F403
     IntensityNormalizationTask,
-    results_to_tomoscan_norm,
 )
 from .params import IntensityNormalizationParams  # noqa F403

tomwer/core/process/reconstruction/normalization/normalization.py

@@ -37,7 +37,6 @@ __date__ = "25/06/2021"
 from tomwer.core.process.task import Task
 from .params import (
     IntensityNormalizationParams,
-    Method,
     _CalculationArea,
     _ValueCalculationMethod,
     _ValueCalculationFct,
@@ -51,10 +50,8 @@ import tomoscan.esrf.utils
 import tomwer.version
 import silx.io.utils
 import functools
-import typing
 import numpy
 import logging
-from tomoscan.normalization import Method as TomoScanMethod
 
 
 _logger = logging.getLogger(__name__)
@@ -104,40 +101,52 @@ class IntensityNormalizationTask(
             return
         scan.intensity_normalization.tomwer_processing_res_code = None
         params = IntensityNormalizationParams.from_dict(self._settings)
+        # define the method used to the scan
+        scan.intensity_normalization.method = params.method
+        # after this processing the source
         try:
             if params.source is _ValueSource.MANUAL_ROI:
-                res = self._compute_from_manual_roi(scan)
+                value = self._compute_from_manual_roi(scan)
                 # need_conversion_to_tomoscan = True
+                # insure this could be hashable (for caches)
+                if isinstance(value, numpy.ndarray):
+                    value = tuple(value)
+                final_norm_info = {
+                    "value": value,
+                    "source": _ValueSource.MANUAL_SCALAR.value,
+                }
             elif params.source is _ValueSource.AUTO_ROI:
-                res = self._compute_from_automatic_roi(scan)
+                value = self._compute_from_automatic_roi(scan)
                 # need_conversion_to_tomoscan = True
+                # insure this could be hashable (for caches)
+                if isinstance(value, numpy.ndarray):
+                    value = tuple(value)
+                final_norm_info = {
+                    "value": value,
+                    "source": _ValueSource.MANUAL_SCALAR.value,
+                }
             elif params.source is _ValueSource.DATASET:
-                res = self._compute_from_dataset()
-                # need_conversion_to_tomoscan = True
-            elif params.method.value in TomoScanMethod.values():
-                # need_conversion_to_tomoscan = False
-                res = None
+                final_norm_info = {
+                    "dataset_url": params.extra_infos.get("dataset_url", None),
+                    "source": _ValueSource.DATASET.value,
+                }
+            elif params.source is _ValueSource.MANUAL_SCALAR:
+                final_norm_info = {
+                    "value": params.extra_infos.get("value", None),
+                    "source": _ValueSource.MANUAL_SCALAR.value,
+                }
             else:
                 raise ValueError("method {} is not handled".format(params.method))
         except Exception as e:
             _logger.error(e)
-            scan.intensity_normalization.tomwer_processing_res_code = False
-            res = None
+            tomwer_processing_res_code = False
+            final_norm_info = {}
         else:
-            scan.intensity_normalization.tomwer_processing_res_code = True
-        # insure this could be hashable (for caches)
-        if isinstance(res, numpy.ndarray):
-            res = tuple(res)
-
-        scan.intensity_normalization.tomwer_processing_res = res
-        # if need_conversion_to_tomoscan:
-        #     results_to_tomoscan_norm(
-        #         scan=scan,
-        #         method=params.method,
-        #         results=res,
-        #         extra_infos=params.extra_infos,
-        #     )
-
+            tomwer_processing_res_code = True
+        scan.intensity_normalization.set_extra_infos(final_norm_info)
+        scan.intensity_normalization.tomwer_processing_res_code = (
+            tomwer_processing_res_code
+        )
         self.outputs.data = scan
 
     def _compute_from_manual_roi(self, scan):
@@ -169,7 +178,7 @@ class IntensityNormalizationTask(
         #                      "computing ROI on a single projection")
 
         try:
-            return self._cache_compute_from_manual_roi(
+            value = self._cache_compute_from_manual_roi(
                 dataset_identifier=scan.get_dataset_identifier(),
                 start_x=start_x,
                 start_y=start_y,
@@ -180,6 +189,8 @@ class IntensityNormalizationTask(
         except Exception as e:
             _logger.error(e)
             return None
+        else:
+            return value
 
     @staticmethod
     @functools.lru_cache(

tomwer/core/process/reconstruction/normalization/params.py

@@ -112,18 +112,13 @@ class IntensityNormalizationParams:
     """Information regarding the intensity normalization to be done"""
 
     def __init__(self, method=Method.NONE, source=_ValueSource.NONE, extra_infos=None):
-        if not isinstance(method, (str, Method)):
-            raise TypeError(
-                "method is expected to be a str or an instance " "of Method"
-            )
-        if not isinstance(extra_infos, (type(None), dict)):
-            raise TypeError(
-                "extra_infos is expected to be None or a dict not "
-                "{}".format(extra_infos)
-            )
-        self._method = Method.from_value(method)
-        self._extra_infos = extra_infos if extra_infos is not None else {}
-        self._source = _ValueSource.from_value(source)
+        self._method = Method.NONE
+        self._source = _ValueSource.NONE
+        self._extra_infos = {}
+
+        self.method = method
+        self.extra_infos = extra_infos if extra_infos is not None else {}
+        self.source = source
 
     @property
     def method(self):

tomwer/core/scan/scanbase.py

@@ -38,6 +38,7 @@ from glob import glob
 from silx.io.url import DataUrl
 from tomwer.core.utils.locker import FileLockerManager, FileLockerContext
 from tomwer.core.utils.ftseriesutils import orderFileByLastLastModification
+from tomoscan.normalization import IntensityNormalization
 from tomoscan.io import HDF5File
 from processview.core.dataset import Dataset
 from typing import Optional
@@ -69,6 +70,8 @@ class TomwerScanBase(Dataset):
 
     _DICT_PROCESS_INDEX_KEY = "next_process_index"
 
+    _DICT_NORMALIZATION_KEY = "norm_params"
+
     VALID_RECONS_EXTENSION = ".edf", ".npy", ".npz", ".hdf5", ".tiff", ".jp2"
 
     def __init__(self, overwrite_proc_file=False):
@@ -455,6 +458,11 @@ class TomwerScanBase(Dataset):
             res[self._DICT_DARK_REF_KEYS] = None
         else:
             res[self._DICT_DARK_REF_KEYS] = self._dark_ref_params.to_dict()
+        # normalization
+        if self.intensity_normalization is None:
+            res[self._DICT_NORMALIZATION_KEY] = None
+        else:
+            res[self._DICT_NORMALIZATION_KEY] = self.intensity_normalization.to_dict()
         # process index
         res[self._DICT_PROCESS_INDEX_KEY] = self._process_index
 
@@ -497,6 +505,12 @@ class TomwerScanBase(Dataset):
             from tomwer.core.process.reconstruction.darkref.params import DKRFRP
 
             self._dark_ref_params = DKRFRP.from_dict(dark_ref_params)
+        # load normalization
+        intensity_normalization = data.get(self._DICT_NORMALIZATION_KEY, None)
+        if intensity_normalization is not None:
+            self.intensity_normalization = IntensityNormalization.from_dict(
+                intensity_normalization
+            )
         # load saaxis parameters
         saaxis_params = data.get(self._DICT_SA_AXIS_KEYS, None)
         if saaxis_params is not None:

tomwer/gui/reconstruction/normalization/intensity.py

@@ -45,10 +45,9 @@ from tomwer.gui.visualization.dataviewer import DataViewer
 from tomwer.gui.reconstruction.scores.control import ControlWidget
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.gui.visualization.sinogramviewer import SinogramViewer as _SinogramViewer
-from tomwer.core.process.reconstruction.normalization.normalization import Method
 from tomwer.core.process.reconstruction.normalization import params as _normParams
 from tomwer.core.process.reconstruction.normalization.params import _ValueSource
-from tomoscan.normalization import Method as TomoScanMethod
+from tomoscan.normalization import Method
 from tomwer.gui.utils.buttons import PadlockButton
 import weakref
 import typing
@@ -97,6 +96,11 @@ class NormIntensityWindow(qt.QMainWindow):
         self._centralWidget._updateSinogramROI()
         self._modeChanged()
 
+    def close(self):
+        self._centralWidget.stop()
+        self._centralWidget = None
+        super().close()
+
     def _configurationChanged(self):
         self.sigConfigurationChanged.emit()
 
@@ -112,9 +116,15 @@ class NormIntensityWindow(qt.QMainWindow):
     def setConfiguration(self, config: dict):
         self._optsWidget.setConfiguration(config=config)
 
+    def setCurrentMethod(self, method):
+        self._optsWidget.setCurrentMethod(method=method)
+
     def getCurrentMethod(self):
         return self._optsWidget.getCurrentMethod()
 
+    def setCurrentSource(self, source):
+        self._optsWidget.setCurrentSource(source=source)
+
     def getCurrentSource(self):
         return self._optsWidget.getCurrentSource()
 
@@ -152,9 +162,6 @@ class NormIntensityWindow(qt.QMainWindow):
         self._centralWidget.setScan(scan=scan)
         self._optsWidget.setScan(scan=scan)
 
-    def setCurrentMethod(self, mode):
-        self._optsWidget.setCurrentMethod(mode)
-
     def stop(self):
         if self._centralWidget is not None:
             self._centralWidget.stop()
@@ -175,7 +182,7 @@ class NormIntensityWindow(qt.QMainWindow):
         # once computed update scan normalization values (
         # update scalar value for example)
         scan = self.getScan()
-        if scan and self.getCurrentMethod().value in TomoScanMethod.values():
+        if scan and self.getCurrentMethod().value in Method.values():
             scan.intensity_normalization = self.getCurrentMethod().value
             scan.intensity_normalization.set_extra_infos(self.getExtraArgs())
         self._crtWidget.setResult(result)
@@ -448,7 +455,7 @@ class _NormIntensityOptions(qt.QWidget):
             if mode == _ValueSource.NONE:
                 # filter this value because does not have much sense for the GUI
                 continue
-            if mode == _ValueSource.AUTO_ROI:
+            if mode in (_ValueSource.AUTO_ROI, _ValueSource.MONITOR):
                 continue
             self._sourceCB.addItem(mode.value)
         self._sourceLabel = qt.QLabel("source:", self)
@@ -551,7 +558,10 @@ class _NormIntensityOptions(qt.QWidget):
         self._modeCB.setCurrentIndex(idx)
 
     def getCurrentSource(self):
-        return _ValueSource.from_value(self._sourceCB.currentText())
+        if self.getCurrentMethod() in (Method.DIVISION, Method.SUBSTRACTION):
+            return _ValueSource.from_value(self._sourceCB.currentText())
+        else:
+            return _ValueSource.NONE