Commit ffbd4d46 authored by Henri Payno's avatar Henri Payno
Browse files

normalization: test + bug fixes

parent 74b1c6c1
......@@ -72,6 +72,7 @@ tomwer:
- python -m pip install .[full_no_cuda]
- python -m pip install PyQt5==5.11.3
- python -m pip install pyqtgraph==0.11.0
- python -m pip install git+https://gitlab.esrf.fr/tomotools/tomoscan@update_normalization
- python -m pip install pytest-cov
- python -m pip install git+https://gitlab.esrf.fr/tomotools/nabu
- python -m tomwer --help
......@@ -93,6 +94,7 @@ test:test-tomwer-tutorials:
- python -m pip install fabio --upgrade --pre
- python -m pip install silx --upgrade --pre
- python -m pip install .[full_no_cuda]
- python -m pip install git+https://gitlab.esrf.fr/tomotools/tomoscan@update_normalization
- mkdir tmp_dir
- cd tmp_dir
script:
......
......@@ -156,8 +156,10 @@ class NormIOW(WidgetLongProcessing, SuperviseOW):
self._layout.setContentsMargins(0, 0, 0, 0)
self._layout.addWidget(self._window)
self.loadSettings()
try:
self.loadSettings()
except Exception as e:
_logger.warning(f"Failed to load settings: {e}")
# connect signal / slot
self._window.sigConfigurationChanged.connect(self._updateSettings)
......
......@@ -39,10 +39,16 @@ from tomwer.core.cluster.cluster import (
)
from tomwer.core.process.reconstruction.nabu.target import Target
from tomwer.core.process.reconstruction.nabu.utils import _NabuPhaseMethod
from tomwer.core.scan.edfscan import EDFTomoScan
from tomwer.core.scan.hdf5scan import HDF5TomoScan
from tomwer.core.scan.scanbase import TomwerScanBase
from tomwer.core.utils.slurm import is_slurm_available
from tomwer.core.process.reconstruction.normalization.params import (
_ValueSource as INormSource,
)
from tomoscan.normalization import Method as INormMethod
from silx.utils.enum import Enum as _Enum
from silx.io.url import DataUrl
from .slurm import _exec_nabu_on_slurm
from typing import Iterable, Optional
from typing import Union
......@@ -52,6 +58,8 @@ from . import settings
from . import utils
from time import sleep
import os
import uuid
from tomoscan.io import HDF5File
_logger = logging.getLogger(__name__)
try:
......@@ -468,19 +476,67 @@ class _NabuBaseReconstructor:
config["preproc"] = {}
if self.scan.intensity_normalization.method is INormMethod.NONE:
config["preproc"]["sino_normalization"] = ""
elif self.scan.intensity_normalization.method is INormMethod.SCALAR:
config["preproc"]["sino_normalization"] = "dataset"
# TODO: save scalar in a .hdf5 to pass it to Nabu
raise NotImplementedError("Scalar not implemented at nabu side for now")
elif self.scan.intensity_normalization.method in (
INormMethod.CHEBYSHEV,
INormMethod.LSQR_SPLINE,
INormMethod.MONITOR,
):
else:
config["preproc"][
"sino_normalization"
] = self.scan.intensity_normalization.method.value
extra_infos = self.scan.intensity_normalization.get_extra_infos()
nabu_cfg_folders = os.path.join(
config["output"]["location"], settings.NABU_CFG_FILE_FOLDER
)
os.makedirs(nabu_cfg_folders, exist_ok=True)
serving_hatch_file = os.path.join(
nabu_cfg_folders, "nabu_tomwer_serving_hatch.h5"
)
source = extra_infos.get("source", INormSource.NONE)
source = INormSource.from_value(source)
if source is INormSource.NONE:
pass
elif source is INormSource.MANUAL_SCALAR:
if "value" not in extra_infos:
raise KeyError(
"value should be provided in extra)infos for scalar defined manually"
)
else:
# save the value to a dedicated path in "nabu_tomwer_serving_hatch"
if isinstance(self.scan, HDF5TomoScan):
entry_path = self.scan.entry
elif isinstance(self.scan, EDFTomoScan):
entry_path = "entry"
else:
raise TypeError
with HDF5File(serving_hatch_file, mode="a") as h5f:
serving_hatch_data_path = None
# create a unique dataset path to avoid possible conflicts
while (
serving_hatch_data_path is None
or serving_hatch_data_path in h5f
):
serving_hatch_data_path = "/".join(
[entry_path, str(uuid.uuid1())]
)
h5f[serving_hatch_data_path] = extra_infos["value"]
serving_hatch_url = DataUrl(
file_path="nabu_tomwer_serving_hatch.h5", # configuration file and nabu_tomwer_serving_hatch are in the same folder
data_path=serving_hatch_data_path,
scheme="silx",
)
config["preproc"]["sino_normalization_file"] = serving_hatch_url.path()
elif source is INormSource.DATASET:
url = extra_infos["dataset_url"]
if not isinstance(url, DataUrl):
raise TypeError(
f"dataset_url is expected to be an instance of DataUrl. Not {type(url)}"
)
config["preproc"]["sino_normalization_file"] = url.path()
else:
raise NotImplementedError(f"source type {source.value} is not handled")
return config
def _get_file_basename_reconstruction(self, pag, db):
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "29/03/2022"
import configparser
from silx.io.url import DataUrl
from collections import namedtuple
import pytest
from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSlices
from tomwer.core.process.reconstruction.normalization.params import (
IntensityNormalizationParams,
)
from tomwer.core.process.reconstruction.normalization.normalization import (
IntensityNormalizationTask,
)
from tomwer.core.utils.scanutils import MockHDF5
from tomwer.core.process.reconstruction.nabu import settings as nabu_settings
from tomoscan.io import HDF5File
import numpy
import os
_norm_setting = namedtuple("_norm_setting", ["method", "source", "extra_infos"])
_nabu_norm_field = namedtuple(
"_nabu_norm_field",
["sino_normalization", "has_sino_normalization_file"],
)
normalization_config_test = (
# no normalization
(
_norm_setting(method="none", source=None, extra_infos=None),
_nabu_norm_field(
sino_normalization="",
has_sino_normalization_file=False,
),
),
# chebyshev normalization
(
_norm_setting(method="chebyshev", source=None, extra_infos=None),
_nabu_norm_field(
sino_normalization="chebyshev",
has_sino_normalization_file=False,
),
),
# divide by a scalar
(
_norm_setting(method="division", source="scalar", extra_infos={"value": 12.0}),
_nabu_norm_field(
sino_normalization="division",
has_sino_normalization_file=True,
),
),
# substract a roi
(
_norm_setting(
method="substraction",
source="manual ROI",
extra_infos={
"start_x": 0.0,
"end_x": 1.0,
"start_y": 0.0,
"end_y": 1.0,
"calc_fct": "mean",
"calc_area": "volume",
"calc_method": "scalar",
},
),
_nabu_norm_field(
sino_normalization="substraction",
has_sino_normalization_file=True,
),
),
# substract from a dataset
(
_norm_setting(
method="substraction",
source="from dataset",
extra_infos={
"dataset_url": DataUrl(
file_path="random_dataset.hdf5",
data_path="data",
scheme="silx",
),
"calc_fct": "median",
"calc_area": "volume",
"calc_method": "scalar",
},
),
_nabu_norm_field(
sino_normalization="substraction",
has_sino_normalization_file=True,
),
),
)
@pytest.mark.parametrize(
"norm_setting, expected_nabu_conf", [item for item in normalization_config_test]
)
def test_normalization(norm_setting, expected_nabu_conf, tmp_path):
"""
Insure normalization is correctly provided to nabu configuration file
For this run a normalization process followed by a nabu process.
"""
scan_dir = tmp_path / "scan"
scan_dir.mkdir()
nabu_cfg_folders = os.path.join(scan_dir, nabu_settings.NABU_CFG_FILE_FOLDER)
os.makedirs(nabu_cfg_folders, exist_ok=True)
random_dataset_file = os.path.join(nabu_cfg_folders, "random_dataset.hdf5")
# create a random dataset if necessary
with HDF5File(random_dataset_file, mode="w") as h5f:
h5f["data"] = numpy.ones((20, 20))
mock = MockHDF5(
scan_path=scan_dir,
n_proj=10,
n_ini_proj=10,
scan_range=180,
dim=20,
)
scan = mock.scan
cfg_folder = os.path.join(str(scan_dir), "nabu_cfg_files")
cfg_file = os.path.join(cfg_folder, "entry_scan.cfg")
assert not os.path.exists(cfg_file)
norm_params = IntensityNormalizationParams(
method=norm_setting.method,
source=norm_setting.source,
extra_infos=norm_setting.extra_infos,
)
normalization = IntensityNormalizationTask(
inputs={
"data": scan,
"configuration": norm_params.to_dict(),
},
varinfo=None,
)
normalization.run()
# insure the method is style valid
assert scan.intensity_normalization.method.value == norm_setting.method
assert hasattr(scan.intensity_normalization, "tomwer_processing_res_code")
process = NabuSlices(
inputs={
"data": scan,
"dry_run": True,
},
varinfo=None,
)
process.run()
assert os.path.exists(cfg_file)
configuration = configparser.ConfigParser(allow_no_value=True)
configuration.read(cfg_file)
preproc_section = configuration["preproc"]
sino_normalization = preproc_section.get("sino_normalization", "")
sino_normalization_file = preproc_section.get("sino_normalization_file", "")
assert sino_normalization == expected_nabu_conf.sino_normalization
if expected_nabu_conf.has_sino_normalization_file:
url = DataUrl(path=sino_normalization_file)
assert url.is_valid()
else:
assert sino_normalization_file == ""
assert hasattr(scan.intensity_normalization, "tomwer_processing_res_code")
from .normalization import ( # noqa F403
IntensityNormalizationTask,
results_to_tomoscan_norm,
)
from .params import IntensityNormalizationParams # noqa F403
......@@ -37,7 +37,6 @@ __date__ = "25/06/2021"
from tomwer.core.process.task import Task
from .params import (
IntensityNormalizationParams,
Method,
_CalculationArea,
_ValueCalculationMethod,
_ValueCalculationFct,
......@@ -51,10 +50,8 @@ import tomoscan.esrf.utils
import tomwer.version
import silx.io.utils
import functools
import typing
import numpy
import logging
from tomoscan.normalization import Method as TomoScanMethod
_logger = logging.getLogger(__name__)
......@@ -104,40 +101,52 @@ class IntensityNormalizationTask(
return
scan.intensity_normalization.tomwer_processing_res_code = None
params = IntensityNormalizationParams.from_dict(self._settings)
# define the method used to the scan
scan.intensity_normalization.method = params.method
# after this processing the source
try:
if params.source is _ValueSource.MANUAL_ROI:
res = self._compute_from_manual_roi(scan)
value = self._compute_from_manual_roi(scan)
# need_conversion_to_tomoscan = True
# insure this could be hashable (for caches)
if isinstance(value, numpy.ndarray):
value = tuple(value)
final_norm_info = {
"value": value,
"source": _ValueSource.MANUAL_SCALAR.value,
}
elif params.source is _ValueSource.AUTO_ROI:
res = self._compute_from_automatic_roi(scan)
value = self._compute_from_automatic_roi(scan)
# need_conversion_to_tomoscan = True
# insure this could be hashable (for caches)
if isinstance(value, numpy.ndarray):
value = tuple(value)
final_norm_info = {
"value": value,
"source": _ValueSource.MANUAL_SCALAR.value,
}
elif params.source is _ValueSource.DATASET:
res = self._compute_from_dataset()
# need_conversion_to_tomoscan = True
elif params.method.value in TomoScanMethod.values():
# need_conversion_to_tomoscan = False
res = None
final_norm_info = {
"dataset_url": params.extra_infos.get("dataset_url", None),
"source": _ValueSource.DATASET.value,
}
elif params.source is _ValueSource.MANUAL_SCALAR:
final_norm_info = {
"value": params.extra_infos.get("value", None),
"source": _ValueSource.MANUAL_SCALAR.value,
}
else:
raise ValueError("method {} is not handled".format(params.method))
except Exception as e:
_logger.error(e)
scan.intensity_normalization.tomwer_processing_res_code = False
res = None
tomwer_processing_res_code = False
final_norm_info = {}
else:
scan.intensity_normalization.tomwer_processing_res_code = True
# insure this could be hashable (for caches)
if isinstance(res, numpy.ndarray):
res = tuple(res)
scan.intensity_normalization.tomwer_processing_res = res
# if need_conversion_to_tomoscan:
# results_to_tomoscan_norm(
# scan=scan,
# method=params.method,
# results=res,
# extra_infos=params.extra_infos,
# )
tomwer_processing_res_code = True
scan.intensity_normalization.set_extra_infos(final_norm_info)
scan.intensity_normalization.tomwer_processing_res_code = (
tomwer_processing_res_code
)
self.outputs.data = scan
def _compute_from_manual_roi(self, scan):
......@@ -169,7 +178,7 @@ class IntensityNormalizationTask(
# "computing ROI on a single projection")
try:
return self._cache_compute_from_manual_roi(
value = self._cache_compute_from_manual_roi(
dataset_identifier=scan.get_dataset_identifier(),
start_x=start_x,
start_y=start_y,
......@@ -180,6 +189,8 @@ class IntensityNormalizationTask(
except Exception as e:
_logger.error(e)
return None
else:
return value
@staticmethod
@functools.lru_cache(
......
......@@ -112,18 +112,13 @@ class IntensityNormalizationParams:
"""Information regarding the intensity normalization to be done"""
def __init__(self, method=Method.NONE, source=_ValueSource.NONE, extra_infos=None):
if not isinstance(method, (str, Method)):
raise TypeError(
"method is expected to be a str or an instance " "of Method"
)
if not isinstance(extra_infos, (type(None), dict)):
raise TypeError(
"extra_infos is expected to be None or a dict not "
"{}".format(extra_infos)
)
self._method = Method.from_value(method)
self._extra_infos = extra_infos if extra_infos is not None else {}
self._source = _ValueSource.from_value(source)
self._method = Method.NONE
self._source = _ValueSource.NONE
self._extra_infos = {}
self.method = method
self.extra_infos = extra_infos if extra_infos is not None else {}
self.source = source
@property
def method(self):
......
......@@ -38,6 +38,7 @@ from glob import glob
from silx.io.url import DataUrl
from tomwer.core.utils.locker import FileLockerManager, FileLockerContext
from tomwer.core.utils.ftseriesutils import orderFileByLastLastModification
from tomoscan.normalization import IntensityNormalization
from tomoscan.io import HDF5File
from processview.core.dataset import Dataset
from typing import Optional
......@@ -69,6 +70,8 @@ class TomwerScanBase(Dataset):
_DICT_PROCESS_INDEX_KEY = "next_process_index"
_DICT_NORMALIZATION_KEY = "norm_params"
VALID_RECONS_EXTENSION = ".edf", ".npy", ".npz", ".hdf5", ".tiff", ".jp2"
def __init__(self, overwrite_proc_file=False):
......@@ -455,6 +458,11 @@ class TomwerScanBase(Dataset):
res[self._DICT_DARK_REF_KEYS] = None
else:
res[self._DICT_DARK_REF_KEYS] = self._dark_ref_params.to_dict()
# normalization
if self.intensity_normalization is None:
res[self._DICT_NORMALIZATION_KEY] = None
else:
res[self._DICT_NORMALIZATION_KEY] = self.intensity_normalization.to_dict()
# process index
res[self._DICT_PROCESS_INDEX_KEY] = self._process_index
......@@ -497,6 +505,12 @@ class TomwerScanBase(Dataset):
from tomwer.core.process.reconstruction.darkref.params import DKRFRP
self._dark_ref_params = DKRFRP.from_dict(dark_ref_params)
# load normalization
intensity_normalization = data.get(self._DICT_NORMALIZATION_KEY, None)
if intensity_normalization is not None:
self.intensity_normalization = IntensityNormalization.from_dict(
intensity_normalization
)
# load saaxis parameters
saaxis_params = data.get(self._DICT_SA_AXIS_KEYS, None)
if saaxis_params is not None:
......
......@@ -45,10 +45,9 @@ from tomwer.gui.visualization.dataviewer import DataViewer
from tomwer.gui.reconstruction.scores.control import ControlWidget
from tomwer.core.scan.scanbase import TomwerScanBase
from tomwer.gui.visualization.sinogramviewer import SinogramViewer as _SinogramViewer
from tomwer.core.process.reconstruction.normalization.normalization import Method
from tomwer.core.process.reconstruction.normalization import params as _normParams
from tomwer.core.process.reconstruction.normalization.params import _ValueSource
from tomoscan.normalization import Method as TomoScanMethod
from tomoscan.normalization import Method
from tomwer.gui.utils.buttons import PadlockButton
import weakref
import typing
......@@ -97,6 +96,11 @@ class NormIntensityWindow(qt.QMainWindow):
self._centralWidget._updateSinogramROI()
self._modeChanged()
def close(self):
self._centralWidget.stop()
self._centralWidget = None
super().close()
def _configurationChanged(self):
self.sigConfigurationChanged.emit()
......@@ -112,9 +116,15 @@ class NormIntensityWindow(qt.QMainWindow):
def setConfiguration(self, config: dict):
self._optsWidget.setConfiguration(config=config)
def setCurrentMethod(self, method):
self._optsWidget.setCurrentMethod(method=method)
def getCurrentMethod(self):
return self._optsWidget.getCurrentMethod()
def setCurrentSource(self, source):
self._optsWidget.setCurrentSource(source=source)
def getCurrentSource(self):
return self._optsWidget.getCurrentSource()
......@@ -152,9 +162,6 @@ class NormIntensityWindow(qt.QMainWindow):
self._centralWidget.setScan(scan=scan)
self._optsWidget.setScan(scan=scan)
def setCurrentMethod(self, mode):
self._optsWidget.setCurrentMethod(mode)
def stop(self):
if self._centralWidget is not None:
self._centralWidget.stop()
......@@ -175,7 +182,7 @@ class NormIntensityWindow(qt.QMainWindow):
# once computed update scan normalization values (
# update scalar value for example)
scan = self.getScan()
if scan and self.getCurrentMethod().value in TomoScanMethod.values():
if scan and self.getCurrentMethod().value in Method.values():
scan.intensity_normalization = self.getCurrentMethod().value
scan.intensity_normalization.set_extra_infos(self.getExtraArgs())
self._crtWidget.setResult(result)
......@@ -448,7 +455,7 @@ class _NormIntensityOptions(qt.QWidget):
if mode == _ValueSource.NONE:
# filter this value because does not have much sense for the GUI
continue
if mode == _ValueSource.AUTO_ROI:
if mode in (_ValueSource.AUTO_ROI, _ValueSource.MONITOR):
continue
self._sourceCB.addItem(mode.value)
self._sourceLabel = qt.QLabel("source:", self)
......@@ -551,7 +558,10 @@ class _NormIntensityOptions(qt.QWidget):
self._modeCB.setCurrentIndex(idx)
def getCurrentSource(self):
return _ValueSource.from_value(self._sourceCB.currentText())
if self.getCurrentMethod() in (Method.DIVISION, Method.SUBSTRACTION):
return _ValueSource.from_value(self._sourceCB.currentText())
else:
return _ValueSource.NONE